Mercurial > libervia-backend
annotate sat/memory/sqlite.py @ 3317:83f25da66bec
core (memory): files are now public by default except for those in root directory:
using public permissions by default makes permissions change more easy as we don't have to
recursively change all permissions of sub files when changing directory. Root directories
are private by default.
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Sat, 01 Aug 2020 16:02:41 +0200 |
| parents | 780fb8dd07ef |
| children | be6d91572633 |
| rev | line source |
|---|---|
| 3028 | 1 #!/usr/bin/env python3 |
| 3137 | 2 |
| 412 | 3 |
|
609
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
4 # SAT: a jabber client |
| 3136 | 5 # Copyright (C) 2009-2020 Jérôme Poisson (goffi@goffi.org) |
| 412 | 6 |
|
609
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
|
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
|
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
|
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
10 # (at your option) any later version. |
| 412 | 11 |
|
609
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
|
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
15 # GNU Affero General Public License for more details. |
| 412 | 16 |
|
609
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
|
84a6e83157c2
fixed licences in docstrings (they are now in comments)
Goffi <goffi@goffi.org>
parents:
593
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 412 | 19 |
| 771 | 20 from sat.core.i18n import _ |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
21 from sat.core.constants import Const as C |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
22 from sat.core import exceptions |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
23 from sat.core.log import getLogger |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
24 from sat.memory.crypto import BlockCipher, PasswordHasher |
|
1045
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
25 from sat.tools.config import fixConfigOption |
| 412 | 26 from twisted.enterprise import adbapi |
| 27 from twisted.internet import defer | |
| 2500 | 28 from twisted.words.protocols.jabber import jid |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
29 from twisted.python import failure |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
30 from collections import OrderedDict |
|
2964
17c61d09a85b
memory (sqlite): exit with an error if current db version is higher than version expected in current code
Goffi <goffi@goffi.org>
parents:
2928
diff
changeset
|
31 import sys |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
32 import re |
| 412 | 33 import os.path |
| 3028 | 34 import pickle as pickle |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
35 import hashlib |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
36 import sqlite3 |
| 2500 | 37 import json |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
38 |
|
2964
17c61d09a85b
memory (sqlite): exit with an error if current db version is higher than version expected in current code
Goffi <goffi@goffi.org>
parents:
2928
diff
changeset
|
39 log = getLogger(__name__) |
|
17c61d09a85b
memory (sqlite): exit with an error if current db version is higher than version expected in current code
Goffi <goffi@goffi.org>
parents:
2928
diff
changeset
|
40 |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
41 CURRENT_DB_VERSION = 9 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
42 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
43 # XXX: DATABASE schemas are used in the following way: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
44 # - 'current' key is for the actual database schema, for a new base |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
45 # - x(int) is for update needed between x-1 and x. All number are needed between y and z to do an update |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
46 # e.g.: if CURRENT_DB_VERSION is 6, 'current' is the actuel DB, and to update from version 3, numbers 4, 5 and 6 are needed |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
47 # a 'current' data dict can contains the keys: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
48 # - 'CREATE': it contains an Ordered dict with table to create as keys, and a len 2 tuple as value, where value[0] are the columns definitions and value[1] are the table constraints |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
49 # - 'INSERT': it contains an Ordered dict with table where values have to be inserted, and many tuples containing values to insert in the order of the rows (#TODO: manage named columns) |
| 2787 | 50 # - 'INDEX': |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
51 # an update data dict (the ones with a number) can contains the keys 'create', 'delete', 'cols create', 'cols delete', 'cols modify', 'insert' or 'specific'. See Updater.generateUpdateData for more infos. This method can be used to autogenerate update_data, to ease the work of the developers. |
| 2500 | 52 # TODO: indexes need to be improved |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
53 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
54 DATABASE_SCHEMAS = { |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
55 "current": {'CREATE': OrderedDict(( |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
56 ('profiles', (("id INTEGER PRIMARY KEY ASC", "name TEXT"), |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
57 ("UNIQUE (name)",))), |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
58 ('components', (("profile_id INTEGER PRIMARY KEY", "entry_point TEXT NOT NULL"), |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
59 ("FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE",))), |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
60 ('message_types', (("type TEXT PRIMARY KEY",), |
| 2500 | 61 ())), |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
62 ('history', (("uid TEXT PRIMARY KEY", "stanza_id TEXT", "update_uid TEXT", "profile_id INTEGER", "source TEXT", "dest TEXT", "source_res TEXT", "dest_res TEXT", |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
63 "timestamp DATETIME NOT NULL", "received_timestamp DATETIME", # XXX: timestamp is the time when the message was emitted. If received time stamp is not NULL, the message was delayed and timestamp is the declared value (and received_timestamp the time of reception) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
64 "type TEXT", "extra BLOB"), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
65 ("FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE", "FOREIGN KEY(type) REFERENCES message_types(type)", |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
66 "UNIQUE (profile_id, stanza_id, source, dest)" # avoid storing 2 times the same message |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
67 ))), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
68 ('message', (("id INTEGER PRIMARY KEY ASC", "history_uid INTEGER", "message TEXT", "language TEXT"), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
69 ("FOREIGN KEY(history_uid) REFERENCES history(uid) ON DELETE CASCADE",))), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
70 ('subject', (("id INTEGER PRIMARY KEY ASC", "history_uid INTEGER", "subject TEXT", "language TEXT"), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
71 ("FOREIGN KEY(history_uid) REFERENCES history(uid) ON DELETE CASCADE",))), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
72 ('thread', (("id INTEGER PRIMARY KEY ASC", "history_uid INTEGER", "thread_id TEXT", "parent_id TEXT"),("FOREIGN KEY(history_uid) REFERENCES history(uid) ON DELETE CASCADE",))), |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
73 ('param_gen', (("category TEXT", "name TEXT", "value TEXT"), |
| 2787 | 74 ("PRIMARY KEY (category, name)",))), |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
75 ('param_ind', (("category TEXT", "name TEXT", "profile_id INTEGER", "value TEXT"), |
| 2787 | 76 ("PRIMARY KEY (profile_id, category, name)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE"))), |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
77 ('private_gen', (("namespace TEXT", "key TEXT", "value TEXT"), |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
78 ("PRIMARY KEY (namespace, key)",))), |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
79 ('private_ind', (("namespace TEXT", "key TEXT", "profile_id INTEGER", "value TEXT"), |
| 2787 | 80 ("PRIMARY KEY (profile_id, namespace, key)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE"))), |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
81 ('private_gen_bin', (("namespace TEXT", "key TEXT", "value BLOB"), |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
82 ("PRIMARY KEY (namespace, key)",))), |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
83 ('private_ind_bin', (("namespace TEXT", "key TEXT", "profile_id INTEGER", "value BLOB"), |
| 2787 | 84 ("PRIMARY KEY (profile_id, namespace, key)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE"))), |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
85 ('files', (("id TEXT NOT NULL", "public_id TEXT", "version TEXT NOT NULL", |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
86 "parent TEXT NOT NULL", |
| 2500 | 87 "type TEXT CHECK(type in ('{file}', '{directory}')) NOT NULL DEFAULT '{file}'".format( |
| 88 file=C.FILE_TYPE_FILE, directory=C.FILE_TYPE_DIRECTORY), | |
|
2507
4c45df43ea44
core (memory/sqlite): renamed column for files from hash to file_hash:
Goffi <goffi@goffi.org>
parents:
2500
diff
changeset
|
89 "file_hash TEXT", "hash_algo TEXT", "name TEXT NOT NULL", "size INTEGER", |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
90 "namespace TEXT", "media_type TEXT", "media_subtype TEXT", |
| 2500 | 91 "created DATETIME NOT NULL", "modified DATETIME", |
| 92 "owner TEXT", "access TEXT", "extra TEXT", "profile_id INTEGER"), | |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
93 ("PRIMARY KEY (id, version)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE", |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
94 "UNIQUE (public_id)"))), |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
95 )), |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
96 'INSERT': OrderedDict(( |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
97 ('message_types', (("'chat'",), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
98 ("'error'",), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
99 ("'groupchat'",), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
100 ("'headline'",), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
101 ("'normal'",), |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
102 ("'info'",) # info is not standard, but used to keep track of info like join/leave in a MUC |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
103 )), |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
104 )), |
| 2787 | 105 'INDEX': (('history', (('profile_id', 'timestamp'), |
| 106 ('profile_id', 'received_timestamp'))), | |
| 107 ('message', ('history_uid',)), | |
| 108 ('subject', ('history_uid',)), | |
| 109 ('thread', ('history_uid',)), | |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
110 ('files', (('profile_id', 'owner', 'media_type', 'media_subtype'), |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
111 ('profile_id', 'owner', 'parent'))), |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
112 ) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
113 }, |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
114 9: {'specific': 'update_v9' |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
115 }, |
| 2787 | 116 8: {'specific': 'update_v8' |
| 117 }, | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
118 7: {'specific': 'update_v7' |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
119 }, |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
120 6: {'cols create': {'history': ('stanza_id TEXT',)}, |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
121 }, |
| 2500 | 122 5: {'create': {'files': (("id TEXT NOT NULL", "version TEXT NOT NULL", "parent TEXT NOT NULL", |
| 123 "type TEXT CHECK(type in ('{file}', '{directory}')) NOT NULL DEFAULT '{file}'".format( | |
| 124 file=C.FILE_TYPE_FILE, directory=C.FILE_TYPE_DIRECTORY), | |
|
2507
4c45df43ea44
core (memory/sqlite): renamed column for files from hash to file_hash:
Goffi <goffi@goffi.org>
parents:
2500
diff
changeset
|
125 "file_hash TEXT", "hash_algo TEXT", "name TEXT NOT NULL", "size INTEGER", |
| 2500 | 126 "namespace TEXT", "mime_type TEXT", |
| 127 "created DATETIME NOT NULL", "modified DATETIME", | |
| 128 "owner TEXT", "access TEXT", "extra TEXT", "profile_id INTEGER"), | |
| 129 ("PRIMARY KEY (id, version)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE"))}, | |
| 130 }, | |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
131 4: {'create': {'components': (('profile_id INTEGER PRIMARY KEY', 'entry_point TEXT NOT NULL'), ('FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE',))} |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
132 }, |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
133 3: {'specific': 'update_v3' |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
134 }, |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
135 2: {'specific': 'update2raw_v2' |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
136 }, |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
137 1: {'cols create': {'history': ('extra BLOB',)}, |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
138 }, |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
139 } |
| 412 | 140 |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
141 NOT_IN_EXTRA = ('stanza_id', 'received_timestamp', 'update_uid') # keys which are in message data extra but not stored in sqlite's extra field |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
142 # this is specific to this sqlite storage and for now only used for received_timestamp |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
143 # because this value is stored in a separate field |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
144 |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
145 |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
146 class ConnectionPool(adbapi.ConnectionPool): |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
147 def _runQuery(self, trans, *args, **kw): |
|
2357
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
148 retry = kw.pop('query_retry', 6) |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
149 try: |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
150 trans.execute(*args, **kw) |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
151 except sqlite3.IntegrityError as e: |
|
2747
6487b8855c9a
memory (sqlite): minor comment move
Goffi <goffi@goffi.org>
parents:
2746
diff
changeset
|
152 # Workaround to avoid IntegrityError causing (i)pdb to be |
|
6487b8855c9a
memory (sqlite): minor comment move
Goffi <goffi@goffi.org>
parents:
2746
diff
changeset
|
153 # launched in debug mode |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
154 raise failure.Failure(e) |
|
2357
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
155 except Exception as e: |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
156 # FIXME: in case of error, we retry a couple of times |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
157 # this is a workaround, we need to move to better |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
158 # Sqlite integration, probably with high level library |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
159 retry -= 1 |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
160 if retry == 0: |
| 3028 | 161 log.error(_('too many db tries, we abandon! Error message: {msg}\n' |
| 162 'query was {query}' | |
| 163 .format(msg=e, query=' '.join([str(a) for a in args])))) | |
|
2357
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
164 raise e |
|
2746
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
165 log.warning( |
| 3028 | 166 _('exception while running query, retrying ({try_}): {msg}').format( |
|
2357
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
167 try_ = 6 - retry, |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
168 msg = e)) |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
169 kw['query_retry'] = retry |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
170 return self._runQuery(trans, *args, **kw) |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
171 return trans.fetchall() |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
172 |
|
2746
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
173 def _runInteraction(self, interaction, *args, **kw): |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
174 # sometimes interaction may fail while committing in _runInteraction |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
175 # and it may be due to a db lock. So we work around it in a similar way |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
176 # as for _runQuery but with only 3 tries |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
177 retry = kw.pop('interaction_retry', 4) |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
178 try: |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
179 return adbapi.ConnectionPool._runInteraction(self, interaction, *args, **kw) |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
180 except Exception as e: |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
181 retry -= 1 |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
182 if retry == 0: |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
183 log.error( |
| 3028 | 184 _('too many interaction tries, we abandon! Error message: {msg}\n' |
| 185 'interaction method was: {interaction}\n' | |
| 186 'interaction arguments were: {args}' | |
|
2919
e4715a609d75
memory (sqlite): minor log improvment
Goffi <goffi@goffi.org>
parents:
2918
diff
changeset
|
187 .format(msg=e, interaction=interaction, |
| 3028 | 188 args=', '.join([str(a) for a in args])))) |
|
2746
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
189 raise e |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
190 log.warning( |
| 3028 | 191 _('exception while running interaction, retrying ({try_}): {msg}') |
|
2746
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
192 .format(try_ = 4 - retry, msg = e)) |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
193 kw['interaction_retry'] = retry |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
194 return self._runInteraction(interaction, *args, **kw) |
|
1e2f0856c845
memory (sqlite): retry _runInteraction in the same way as for _runQuery, this improve reliability
Goffi <goffi@goffi.org>
parents:
2735
diff
changeset
|
195 |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
196 |
|
588
beaf6bec2fcd
Remove every old-style class.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
587
diff
changeset
|
197 class SqliteStorage(object): |
| 412 | 198 """This class manage storage with Sqlite database""" |
| 199 | |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
200 def __init__(self, db_filename, sat_version): |
| 412 | 201 """Connect to the given database |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
202 |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
203 @param db_filename: full path to the Sqlite database |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
204 """ |
| 3120 | 205 # triggered when memory is fully initialised and ready |
| 206 self.initialized = defer.Deferred() | |
| 207 # we keep cache for the profiles (key: profile name, value: profile id) | |
| 208 self.profiles = {} | |
| 412 | 209 |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
210 log.info(_("Connecting database")) |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
211 new_base = not os.path.exists(db_filename) # do we have to create the database ? |
|
936
6404df5305e3
memory: be sure that local_dir exists before creating a new database
souliane <souliane@mailoo.org>
parents:
934
diff
changeset
|
212 if new_base: # the dir may not exist if it's not the XDG recommended one |
|
6404df5305e3
memory: be sure that local_dir exists before creating a new database
souliane <souliane@mailoo.org>
parents:
934
diff
changeset
|
213 dir_ = os.path.dirname(db_filename) |
|
6404df5305e3
memory: be sure that local_dir exists before creating a new database
souliane <souliane@mailoo.org>
parents:
934
diff
changeset
|
214 if not os.path.exists(dir_): |
| 3028 | 215 os.makedirs(dir_, 0o700) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
216 |
|
2357
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
217 def foreignKeysOn(sqlite): |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
218 sqlite.execute('PRAGMA foreign_keys = ON') |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
219 |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
220 self.dbpool = ConnectionPool("sqlite3", db_filename, cp_openfun=foreignKeysOn, check_same_thread=False, timeout=15) |
|
624
70988f08d0ad
core: fixed bad database creation on first run in sqlite storage
Goffi <goffi@goffi.org>
parents:
609
diff
changeset
|
221 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
222 def getNewBaseSql(): |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
223 log.info(_("The database is new, creating the tables")) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
224 database_creation = ["PRAGMA user_version=%d" % CURRENT_DB_VERSION] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
225 database_creation.extend(Updater.createData2Raw(DATABASE_SCHEMAS['current']['CREATE'])) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
226 database_creation.extend(Updater.insertData2Raw(DATABASE_SCHEMAS['current']['INSERT'])) |
| 2787 | 227 database_creation.extend(Updater.indexData2Raw(DATABASE_SCHEMAS['current']['INDEX'])) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
228 return database_creation |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
229 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
230 def getUpdateSql(): |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
231 updater = Updater(self, sat_version) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
232 return updater.checkUpdates() |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
233 |
|
2357
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
234 # init_defer is the initialisation deferred, initialisation is ok when all its callbacks have been done |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
235 |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
236 init_defer = defer.succeed(None) |
|
fa43e285df1d
core (memory/sqlite): better stability:
Goffi <goffi@goffi.org>
parents:
2209
diff
changeset
|
237 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
238 init_defer.addCallback(lambda ignore: getNewBaseSql() if new_base else getUpdateSql()) |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
239 init_defer.addCallback(self.commitStatements) |
| 412 | 240 |
| 241 def fillProfileCache(ignore): | |
| 3120 | 242 return self.dbpool.runQuery("SELECT profile_id, entry_point FROM components").addCallback(self._cacheComponentsAndProfiles) |
|
587
952322b1d490
Remove trailing whitespaces.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
572
diff
changeset
|
243 |
|
624
70988f08d0ad
core: fixed bad database creation on first run in sqlite storage
Goffi <goffi@goffi.org>
parents:
609
diff
changeset
|
244 init_defer.addCallback(fillProfileCache) |
| 3120 | 245 init_defer.chainDeferred(self.initialized) |
| 412 | 246 |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
247 def commitStatements(self, statements): |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
248 |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
249 if statements is None: |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
250 return defer.succeed(None) |
| 3028 | 251 log.debug("\n===== COMMITTING STATEMENTS =====\n%s\n============\n\n" % '\n'.join(statements)) |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
252 d = self.dbpool.runInteraction(self._updateDb, tuple(statements)) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
253 return d |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
254 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
255 def _updateDb(self, interaction, statements): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
256 for statement in statements: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
257 interaction.execute(statement) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
258 |
| 2500 | 259 ## Profiles |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
260 |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
261 def _cacheComponentsAndProfiles(self, components_result): |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
262 """Get components results and send requests profiles |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
263 |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
264 they will be both put in cache in _profilesCache |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
265 """ |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
266 return self.dbpool.runQuery("SELECT name,id FROM profiles").addCallback( |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
267 self._cacheComponentsAndProfiles2, components_result) |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
268 |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
269 def _cacheComponentsAndProfiles2(self, profiles_result, components): |
| 412 | 270 """Fill the profiles cache |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
271 |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
272 @param profiles_result: result of the sql profiles query |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
273 """ |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
274 self.components = dict(components) |
| 412 | 275 for profile in profiles_result: |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
276 name, id_ = profile |
|
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
277 self.profiles[name] = id_ |
|
587
952322b1d490
Remove trailing whitespaces.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
572
diff
changeset
|
278 |
| 412 | 279 def getProfilesList(self): |
| 280 """"Return list of all registered profiles""" | |
| 3028 | 281 return list(self.profiles.keys()) |
| 412 | 282 |
| 283 def hasProfile(self, profile_name): | |
| 284 """return True if profile_name exists | |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
285 |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
286 @param profile_name: name of the profile to check |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
287 """ |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
288 return profile_name in self.profiles |
|
587
952322b1d490
Remove trailing whitespaces.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
572
diff
changeset
|
289 |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
290 def profileIsComponent(self, profile_name): |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
291 try: |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
292 return self.profiles[profile_name] in self.components |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
293 except KeyError: |
| 3028 | 294 raise exceptions.NotFound("the requested profile doesn't exists") |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
295 |
|
2144
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
296 def getEntryPoint(self, profile_name): |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
297 try: |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
298 return self.components[self.profiles[profile_name]] |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
299 except KeyError: |
| 3028 | 300 raise exceptions.NotFound("the requested profile doesn't exists or is not a component") |
|
2144
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
301 |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
302 def createProfile(self, name, component=None): |
| 412 | 303 """Create a new profile |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
304 |
|
2144
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
305 @param name(unicode): name of the profile |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
306 @param component(None, unicode): if not None, must point to a component entry point |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
307 @return: deferred triggered once profile is actually created |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
308 """ |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
309 |
| 412 | 310 def getProfileId(ignore): |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
311 return self.dbpool.runQuery("SELECT (id) FROM profiles WHERE name = ?", (name, )) |
|
587
952322b1d490
Remove trailing whitespaces.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
572
diff
changeset
|
312 |
|
2144
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
313 def setComponent(profile_id): |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
314 id_ = profile_id[0][0] |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
315 d_comp = self.dbpool.runQuery("INSERT INTO components(profile_id, entry_point) VALUES (?, ?)", (id_, component)) |
|
2765
378188abe941
misc: replaced all "dummy" by the more conventional and readable "__" ("_" being used for gettext)
Goffi <goffi@goffi.org>
parents:
2747
diff
changeset
|
316 d_comp.addCallback(lambda __: profile_id) |
|
2144
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
317 return d_comp |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
318 |
| 412 | 319 def profile_created(profile_id): |
|
2144
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
320 id_= profile_id[0][0] |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
321 self.profiles[name] = id_ # we synchronise the cache |
|
587
952322b1d490
Remove trailing whitespaces.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
572
diff
changeset
|
322 |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
323 d = self.dbpool.runQuery("INSERT INTO profiles(name) VALUES (?)", (name, )) |
| 412 | 324 d.addCallback(getProfileId) |
|
2144
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
325 if component is not None: |
|
1d3f73e065e1
core, jp: component handling + client handling refactoring:
Goffi <goffi@goffi.org>
parents:
2143
diff
changeset
|
326 d.addCallback(setComponent) |
| 412 | 327 d.addCallback(profile_created) |
| 328 return d | |
|
587
952322b1d490
Remove trailing whitespaces.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
572
diff
changeset
|
329 |
|
420
acd908528ef7
core: profile creation/deletion through database
Goffi <goffi@goffi.org>
parents:
416
diff
changeset
|
330 def deleteProfile(self, name): |
|
acd908528ef7
core: profile creation/deletion through database
Goffi <goffi@goffi.org>
parents:
416
diff
changeset
|
331 """Delete profile |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
332 |
|
420
acd908528ef7
core: profile creation/deletion through database
Goffi <goffi@goffi.org>
parents:
416
diff
changeset
|
333 @param name: name of the profile |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
334 @return: deferred triggered once profile is actually deleted |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
335 """ |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
336 def deletionError(failure_): |
| 3028 | 337 log.error(_("Can't delete profile [%s]") % name) |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
338 return failure_ |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
339 |
|
934
34dd9287dfe5
plugin account: bug fix profile deletion from the database + unsubscribe the contacts
souliane <souliane@mailoo.org>
parents:
853
diff
changeset
|
340 def delete(txn): |
|
1705
3765e10ec52f
core (sqlite storage): do explicit delete to profile linked tables for deleteProfile
Goffi <goffi@goffi.org>
parents:
1455
diff
changeset
|
341 profile_id = self.profiles.pop(name) |
|
934
34dd9287dfe5
plugin account: bug fix profile deletion from the database + unsubscribe the contacts
souliane <souliane@mailoo.org>
parents:
853
diff
changeset
|
342 txn.execute("DELETE FROM profiles WHERE name = ?", (name,)) |
|
1705
3765e10ec52f
core (sqlite storage): do explicit delete to profile linked tables for deleteProfile
Goffi <goffi@goffi.org>
parents:
1455
diff
changeset
|
343 # FIXME: the following queries should be done by the ON DELETE CASCADE |
|
3765e10ec52f
core (sqlite storage): do explicit delete to profile linked tables for deleteProfile
Goffi <goffi@goffi.org>
parents:
1455
diff
changeset
|
344 # but it seems they are not, so we explicitly do them by security |
|
3765e10ec52f
core (sqlite storage): do explicit delete to profile linked tables for deleteProfile
Goffi <goffi@goffi.org>
parents:
1455
diff
changeset
|
345 # this need more investigation |
|
3765e10ec52f
core (sqlite storage): do explicit delete to profile linked tables for deleteProfile
Goffi <goffi@goffi.org>
parents:
1455
diff
changeset
|
346 txn.execute("DELETE FROM history WHERE profile_id = ?", (profile_id,)) |
|
3765e10ec52f
core (sqlite storage): do explicit delete to profile linked tables for deleteProfile
Goffi <goffi@goffi.org>
parents:
1455
diff
changeset
|
347 txn.execute("DELETE FROM param_ind WHERE profile_id = ?", (profile_id,)) |
|
3765e10ec52f
core (sqlite storage): do explicit delete to profile linked tables for deleteProfile
Goffi <goffi@goffi.org>
parents:
1455
diff
changeset
|
348 txn.execute("DELETE FROM private_ind WHERE profile_id = ?", (profile_id,)) |
|
3765e10ec52f
core (sqlite storage): do explicit delete to profile linked tables for deleteProfile
Goffi <goffi@goffi.org>
parents:
1455
diff
changeset
|
349 txn.execute("DELETE FROM private_ind_bin WHERE profile_id = ?", (profile_id,)) |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
350 txn.execute("DELETE FROM components WHERE profile_id = ?", (profile_id,)) |
|
934
34dd9287dfe5
plugin account: bug fix profile deletion from the database + unsubscribe the contacts
souliane <souliane@mailoo.org>
parents:
853
diff
changeset
|
351 return None |
|
34dd9287dfe5
plugin account: bug fix profile deletion from the database + unsubscribe the contacts
souliane <souliane@mailoo.org>
parents:
853
diff
changeset
|
352 |
|
34dd9287dfe5
plugin account: bug fix profile deletion from the database + unsubscribe the contacts
souliane <souliane@mailoo.org>
parents:
853
diff
changeset
|
353 d = self.dbpool.runInteraction(delete) |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
354 d.addCallback(lambda ignore: log.info(_("Profile [%s] deleted") % name)) |
|
593
70bae685d05c
core: added forgotten errback in sqlite's deleteProfile
Goffi <goffi@goffi.org>
parents:
592
diff
changeset
|
355 d.addErrback(deletionError) |
|
420
acd908528ef7
core: profile creation/deletion through database
Goffi <goffi@goffi.org>
parents:
416
diff
changeset
|
356 return d |
|
acd908528ef7
core: profile creation/deletion through database
Goffi <goffi@goffi.org>
parents:
416
diff
changeset
|
357 |
| 2500 | 358 ## Params |
| 412 | 359 def loadGenParams(self, params_gen): |
| 360 """Load general parameters | |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
361 |
| 412 | 362 @param params_gen: dictionary to fill |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
363 @return: deferred |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
364 """ |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
365 |
| 412 | 366 def fillParams(result): |
|
413
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
367 for param in result: |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
368 category, name, value = param |
|
413
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
369 params_gen[(category, name)] = value |
| 3028 | 370 log.debug(_("loading general parameters from database")) |
| 412 | 371 return self.dbpool.runQuery("SELECT category,name,value FROM param_gen").addCallback(fillParams) |
| 372 | |
| 373 def loadIndParams(self, params_ind, profile): | |
|
432
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
374 """Load individual parameters |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
375 |
| 412 | 376 @param params_ind: dictionary to fill |
| 377 @param profile: a profile which *must* exist | |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
378 @return: deferred |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
379 """ |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
380 |
| 412 | 381 def fillParams(result): |
|
413
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
382 for param in result: |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
383 category, name, value = param |
|
423
6c20c76abdcc
backend: - bridge async D-Bus method now automatically manage callback and errback, we just have to return a deferred
Goffi <goffi@goffi.org>
parents:
420
diff
changeset
|
384 params_ind[(category, name)] = value |
| 3028 | 385 log.debug(_("loading individual parameters from database")) |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
386 d = self.dbpool.runQuery("SELECT category,name,value FROM param_ind WHERE profile_id=?", (self.profiles[profile], )) |
| 412 | 387 d.addCallback(fillParams) |
| 388 return d | |
| 389 | |
|
413
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
390 def getIndParam(self, category, name, profile): |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
391 """Ask database for the value of one specific individual parameter |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
392 |
|
413
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
393 @param category: category of the parameter |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
394 @param name: name of the parameter |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
395 @param profile: %(doc_profile)s |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
396 @return: deferred |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
397 """ |
| 3120 | 398 d = self.dbpool.runQuery( |
| 399 "SELECT value FROM param_ind WHERE category=? AND name=? AND profile_id=?", | |
| 400 (category, name, self.profiles[profile])) | |
|
413
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
401 d.addCallback(self.__getFirstResult) |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
402 return d |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
403 |
| 3120 | 404 async def getIndParamValues(self, category, name): |
| 405 """Ask database for the individual values of a parameter for all profiles | |
| 406 | |
| 407 @param category: category of the parameter | |
| 408 @param name: name of the parameter | |
| 409 @return dict: profile => value map | |
| 410 """ | |
| 411 result = await self.dbpool.runQuery( | |
| 412 "SELECT profiles.name, param_ind.value FROM param_ind JOIN profiles ON " | |
| 413 "param_ind.profile_id = profiles.id WHERE param_ind.category=? " | |
| 414 "and param_ind.name=?", | |
| 415 (category, name)) | |
| 416 return dict(result) | |
| 417 | |
| 412 | 418 def setGenParam(self, category, name, value): |
| 419 """Save the general parameters in database | |
| 2500 | 420 |
| 412 | 421 @param category: category of the parameter |
| 422 @param name: name of the parameter | |
| 423 @param value: value to set | |
| 424 @return: deferred""" | |
| 425 d = self.dbpool.runQuery("REPLACE INTO param_gen(category,name,value) VALUES (?,?,?)", (category, name, value)) | |
| 3028 | 426 d.addErrback(lambda ignore: log.error(_("Can't set general parameter (%(category)s/%(name)s) in database" % {"category": category, "name": name}))) |
| 412 | 427 return d |
| 428 | |
| 429 def setIndParam(self, category, name, value, profile): | |
|
432
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
430 """Save the individual parameters in database |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
431 |
| 412 | 432 @param category: category of the parameter |
| 433 @param name: name of the parameter | |
| 434 @param value: value to set | |
| 435 @param profile: a profile which *must* exist | |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
436 @return: deferred |
|
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
437 """ |
| 412 | 438 d = self.dbpool.runQuery("REPLACE INTO param_ind(category,name,profile_id,value) VALUES (?,?,?,?)", (category, name, self.profiles[profile], value)) |
| 3028 | 439 d.addErrback(lambda ignore: log.error(_("Can't set individual parameter (%(category)s/%(name)s) for [%(profile)s] in database" % {"category": category, "name": name, "profile": profile}))) |
| 412 | 440 return d |
|
413
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
441 |
| 2500 | 442 ## History |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
443 |
|
2765
378188abe941
misc: replaced all "dummy" by the more conventional and readable "__" ("_" being used for gettext)
Goffi <goffi@goffi.org>
parents:
2747
diff
changeset
|
444 def _addToHistoryCb(self, __, data): |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
445 # Message metadata were successfuly added to history |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
446 # now we can add message and subject |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
447 uid = data['uid'] |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
448 d_list = [] |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
449 for key in ('message', 'subject'): |
| 3028 | 450 for lang, value in data[key].items(): |
|
3191
7255286a298a
memory (sqlite): don't store empty messages
Goffi <goffi@goffi.org>
parents:
3160
diff
changeset
|
451 if not value.strip(): |
|
7255286a298a
memory (sqlite): don't store empty messages
Goffi <goffi@goffi.org>
parents:
3160
diff
changeset
|
452 # no need to store empty messages |
|
7255286a298a
memory (sqlite): don't store empty messages
Goffi <goffi@goffi.org>
parents:
3160
diff
changeset
|
453 continue |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
454 d = self.dbpool.runQuery( |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
455 "INSERT INTO {key}(history_uid, {key}, language) VALUES (?,?,?)" |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
456 .format(key=key), |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
457 (uid, value, lang or None)) |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
458 d.addErrback(lambda __: log.error( |
| 3028 | 459 _("Can't save following {key} in history (uid: {uid}, lang:{lang}):" |
| 460 " {value}").format( | |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
461 key=key, uid=uid, lang=lang, value=value))) |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
462 d_list.append(d) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
463 try: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
464 thread = data['extra']['thread'] |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
465 except KeyError: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
466 pass |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
467 else: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
468 thread_parent = data['extra'].get('thread_parent') |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
469 d = self.dbpool.runQuery( |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
470 "INSERT INTO thread(history_uid, thread_id, parent_id) VALUES (?,?,?)", |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
471 (uid, thread, thread_parent)) |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
472 d.addErrback(lambda __: log.error( |
| 3028 | 473 _("Can't save following thread in history (uid: {uid}): thread: " |
| 474 "{thread}), parent:{parent}").format( | |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
475 uid=uid, thread=thread, parent=thread_parent))) |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
476 d_list.append(d) |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
477 return defer.DeferredList(d_list) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
478 |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
479 def _addToHistoryEb(self, failure_, data): |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
480 failure_.trap(sqlite3.IntegrityError) |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
481 sqlite_msg = failure_.value.args[0] |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
482 if "UNIQUE constraint failed" in sqlite_msg: |
| 3028 | 483 log.debug("message {} is already in history, not storing it again" |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
484 .format(data['uid'])) |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
485 if 'received_timestamp' not in data: |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
486 log.warning( |
| 3028 | 487 "duplicate message is not delayed, this is maybe a bug: data={}" |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
488 .format(data)) |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
489 # we cancel message to avoid sending duplicate message to frontends |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
490 raise failure.Failure(exceptions.CancelError("Cancelled duplicated message")) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
491 else: |
| 3028 | 492 log.error("Can't store message in history: {}".format(failure_)) |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
493 |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
494 def _logHistoryError(self, failure_, from_jid, to_jid, data): |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
495 if failure_.check(exceptions.CancelError): |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
496 # we propagate CancelError to avoid sending message to frontends |
|
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
497 raise failure_ |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
498 log.error(_( |
| 3028 | 499 "Can't save following message in history: from [{from_jid}] to [{to_jid}] " |
| 500 "(uid: {uid})") | |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
501 .format(from_jid=from_jid.full(), to_jid=to_jid.full(), uid=data['uid'])) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
502 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
503 def addToHistory(self, data, profile): |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
504 """Store a new message in history |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
505 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
506 @param data(dict): message data as build by SatMessageProtocol.onMessage |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
507 """ |
| 3028 | 508 extra = pickle.dumps({k: v for k, v in data['extra'].items() |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
509 if k not in NOT_IN_EXTRA}, 0) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
510 from_jid = data['from'] |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
511 to_jid = data['to'] |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
512 d = self.dbpool.runQuery( |
| 3028 | 513 "INSERT INTO history(uid, stanza_id, update_uid, profile_id, source, dest, " |
| 514 "source_res, dest_res, timestamp, received_timestamp, type, extra) VALUES " | |
| 515 "(?,?,?,?,?,?,?,?,?,?,?,?)", | |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
516 (data['uid'], data['extra'].get('stanza_id'), data['extra'].get('update_uid'), |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
517 self.profiles[profile], data['from'].userhost(), to_jid.userhost(), |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
518 from_jid.resource, to_jid.resource, data['timestamp'], |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
519 data.get('received_timestamp'), data['type'], sqlite3.Binary(extra))) |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
520 d.addCallbacks(self._addToHistoryCb, |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
521 self._addToHistoryEb, |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
522 callbackArgs=[data], |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
523 errbackArgs=[data]) |
|
1961
c73e08094a95
memory (sqlite): better handling of IntegrityError
Goffi <goffi@goffi.org>
parents:
1955
diff
changeset
|
524 d.addErrback(self._logHistoryError, from_jid, to_jid, data) |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
525 return d |
|
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
526 |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
527 def sqliteHistoryToList(self, query_result): |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
528 """Get SQL query result and return a list of message data dicts""" |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
529 result = [] |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
530 current = {'uid': None} |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
531 for row in reversed(query_result): |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
532 (uid, stanza_id, update_uid, source, dest, source_res, dest_res, timestamp, |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
533 received_timestamp, type_, extra, message, message_lang, subject, |
|
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
534 subject_lang, thread, thread_parent) = row |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
535 if uid != current['uid']: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
536 # new message |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
537 try: |
| 3028 | 538 extra = pickle.loads(extra or b"") |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
539 except EOFError: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
540 extra = {} |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
541 current = { |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
542 'from': "%s/%s" % (source, source_res) if source_res else source, |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
543 'to': "%s/%s" % (dest, dest_res) if dest_res else dest, |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
544 'uid': uid, |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
545 'message': {}, |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
546 'subject': {}, |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
547 'type': type_, |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
548 'extra': extra, |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
549 'timestamp': timestamp, |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
550 } |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
551 if stanza_id is not None: |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
552 current['extra']['stanza_id'] = stanza_id |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
553 if update_uid is not None: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
554 current['extra']['update_uid'] = update_uid |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
555 if received_timestamp is not None: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
556 current['extra']['received_timestamp'] = str(received_timestamp) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
557 result.append(current) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
558 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
559 if message is not None: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
560 current['message'][message_lang or ''] = message |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
561 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
562 if subject is not None: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
563 current['subject'][subject_lang or ''] = subject |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
564 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
565 if thread is not None: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
566 current_extra = current['extra'] |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
567 current_extra['thread'] = thread |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
568 if thread_parent is not None: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
569 current_extra['thread_parent'] = thread_parent |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
570 else: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
571 if thread_parent is not None: |
|
3013
860c550028d6
memory (sqlite): properly wait for messages to be writen in database:
Goffi <goffi@goffi.org>
parents:
2989
diff
changeset
|
572 log.error( |
| 3028 | 573 "Database inconsistency: thread parent without thread (uid: " |
| 574 "{uid}, thread_parent: {parent})" | |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
575 .format(uid=uid, parent=thread_parent)) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
576 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
577 return result |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
578 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
579 def listDict2listTuple(self, messages_data): |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
580 """Return a list of tuple as used in bridge from a list of messages data""" |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
581 ret = [] |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
582 for m in messages_data: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
583 ret.append((m['uid'], m['timestamp'], m['from'], m['to'], m['message'], m['subject'], m['type'], m['extra'])) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
584 return ret |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
585 |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
586 def historyGet(self, from_jid, to_jid, limit=None, between=True, filters=None, profile=None): |
|
1222
e6e0ea4dc835
memory: add Parameter "Chat history limit"
souliane <souliane@mailoo.org>
parents:
1221
diff
changeset
|
587 """Retrieve messages in history |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
588 |
|
1222
e6e0ea4dc835
memory: add Parameter "Chat history limit"
souliane <souliane@mailoo.org>
parents:
1221
diff
changeset
|
589 @param from_jid (JID): source JID (full, or bare for catchall) |
|
e6e0ea4dc835
memory: add Parameter "Chat history limit"
souliane <souliane@mailoo.org>
parents:
1221
diff
changeset
|
590 @param to_jid (JID): dest JID (full, or bare for catchall) |
|
e6e0ea4dc835
memory: add Parameter "Chat history limit"
souliane <souliane@mailoo.org>
parents:
1221
diff
changeset
|
591 @param limit (int): maximum number of messages to get: |
|
e6e0ea4dc835
memory: add Parameter "Chat history limit"
souliane <souliane@mailoo.org>
parents:
1221
diff
changeset
|
592 - 0 for no message (returns the empty list) |
|
e6e0ea4dc835
memory: add Parameter "Chat history limit"
souliane <souliane@mailoo.org>
parents:
1221
diff
changeset
|
593 - None for unlimited |
|
e6e0ea4dc835
memory: add Parameter "Chat history limit"
souliane <souliane@mailoo.org>
parents:
1221
diff
changeset
|
594 @param between (bool): confound source and dest (ignore the direction) |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
595 @param filters (dict[unicode, unicode]): pattern to filter the history results |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
596 @param profile (unicode): %(doc_profile)s |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
597 @return: list of tuple as in [messageNew] |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
598 """ |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
599 assert profile |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
600 if filters is None: |
|
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
601 filters = {} |
|
1221
5e5661ab5c81
memory: getHistory with limit=0 now returns an empty list, use limit=None to return all messages
souliane <souliane@mailoo.org>
parents:
1099
diff
changeset
|
602 if limit == 0: |
|
5e5661ab5c81
memory: getHistory with limit=0 now returns an empty list, use limit=None to return all messages
souliane <souliane@mailoo.org>
parents:
1099
diff
changeset
|
603 return defer.succeed([]) |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
604 |
| 3028 | 605 query_parts = ["SELECT uid, stanza_id, update_uid, source, dest, source_res, dest_res, timestamp, received_timestamp,\ |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
606 type, extra, message, message.language, subject, subject.language, thread_id, thread.parent_id\ |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
607 FROM history LEFT JOIN message ON history.uid = message.history_uid\ |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
608 LEFT JOIN subject ON history.uid=subject.history_uid\ |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
609 LEFT JOIN thread ON history.uid=thread.history_uid\ |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
610 WHERE profile_id=?"] # FIXME: not sure if it's the best request, messages and subjects can appear several times here |
|
538
2c4016921403
core, frontends, bridgen plugins: fixed methods which were unproperly managing multi-profiles
Goffi <goffi@goffi.org>
parents:
512
diff
changeset
|
611 values = [self.profiles[profile]] |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
612 |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
613 def test_jid(type_, jid_): |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
614 values.append(jid_.userhost()) |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
615 if jid_.resource: |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
616 values.append(jid_.resource) |
| 3028 | 617 return '({type_}=? AND {type_}_res=?)'.format(type_=type_) |
| 618 return '{type_}=?'.format(type_=type_) | |
|
587
952322b1d490
Remove trailing whitespaces.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
572
diff
changeset
|
619 |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
620 if not from_jid and not to_jid: |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
621 # not jid specified, we want all one2one communications |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
622 pass |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
623 elif between: |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
624 if not from_jid or not to_jid: |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
625 # we only have one jid specified, we check all messages |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
626 # from or to this jid |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
627 jid_ = from_jid or to_jid |
| 3028 | 628 query_parts.append("AND ({source} OR {dest})".format( |
| 629 source=test_jid('source', jid_), | |
| 630 dest=test_jid('dest' , jid_))) | |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
631 else: |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
632 # we have 2 jids specified, we check all communications between |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
633 # those 2 jids |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
634 query_parts.append( |
| 3028 | 635 "AND (({source_from} AND {dest_to}) " |
| 636 "OR ({source_to} AND {dest_from}))".format( | |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
637 source_from=test_jid('source', from_jid), |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
638 dest_to=test_jid('dest', to_jid), |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
639 source_to=test_jid('source', to_jid), |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
640 dest_from=test_jid('dest', from_jid))) |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
641 else: |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
642 # we want one communication in specific direction (from somebody or |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
643 # to somebody). |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
644 q = [] |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
645 if from_jid is not None: |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
646 q.append(test_jid('source', from_jid)) |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
647 if to_jid is not None: |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
648 q.append(test_jid('dest', to_jid)) |
| 3028 | 649 query_parts.append("AND " + " AND ".join(q)) |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
650 |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
651 if filters: |
| 3028 | 652 if 'timestamp_start' in filters: |
| 653 query_parts.append("AND timestamp>= ?") | |
| 654 values.append(float(filters['timestamp_start'])) | |
|
3079
f8cc88c773c8
core (memory/sqlite): added `before_uid` filter to retrieve history before a message UID
Goffi <goffi@goffi.org>
parents:
3028
diff
changeset
|
655 if 'before_uid' in filters: |
|
f8cc88c773c8
core (memory/sqlite): added `before_uid` filter to retrieve history before a message UID
Goffi <goffi@goffi.org>
parents:
3028
diff
changeset
|
656 query_parts.append("AND history.rowid<(select rowid from history where uid=?)") |
|
f8cc88c773c8
core (memory/sqlite): added `before_uid` filter to retrieve history before a message UID
Goffi <goffi@goffi.org>
parents:
3028
diff
changeset
|
657 values.append(filters['before_uid']) |
| 3028 | 658 if 'body' in filters: |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
659 # TODO: use REGEXP (function to be defined) instead of GLOB: https://www.sqlite.org/lang_expr.html |
| 3028 | 660 query_parts.append("AND message LIKE ?") |
| 661 values.append("%{}%".format(filters['body'])) | |
| 662 if 'search' in filters: | |
| 663 query_parts.append("AND (message LIKE ? OR source_res LIKE ?)") | |
| 664 values.extend(["%{}%".format(filters['search'])] * 2) | |
| 665 if 'types' in filters: | |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
666 types = filters['types'].split() |
| 3028 | 667 query_parts.append("AND type IN ({})".format(','.join("?"*len(types)))) |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
668 values.extend(types) |
| 3028 | 669 if 'not_types' in filters: |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
670 types = filters['not_types'].split() |
| 3028 | 671 query_parts.append("AND type NOT IN ({})".format(','.join("?"*len(types)))) |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
672 values.extend(types) |
| 3028 | 673 if 'last_stanza_id' in filters: |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
674 # this request get the last message with a "stanza_id" that we |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
675 # have in history. This is mainly used to retrieve messages sent |
|
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
676 # while we were offline, using MAM (XEP-0313). |
| 3028 | 677 if (filters['last_stanza_id'] is not True |
|
2699
310e41bd6666
core (memory/sqlite): added stanza_id:
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
678 or limit != 1): |
| 3028 | 679 raise ValueError("Unexpected values for last_stanza_id filter") |
| 680 query_parts.append("AND stanza_id IS NOT NULL") | |
|
2013
b536dd121da1
backend (memory), frontends: improved history filtering:
Goffi <goffi@goffi.org>
parents:
1962
diff
changeset
|
681 |
|
2715
b35c84ea73cf
plugin XEP-0045: MAM implementation for MUC
Goffi <goffi@goffi.org>
parents:
2711
diff
changeset
|
682 |
|
2989
c13333fcde5e
memory (sqlite): fixed order for last_stanza_id:
Goffi <goffi@goffi.org>
parents:
2964
diff
changeset
|
683 # timestamp may be identical for 2 close messages (specially when delay is |
|
c13333fcde5e
memory (sqlite): fixed order for last_stanza_id:
Goffi <goffi@goffi.org>
parents:
2964
diff
changeset
|
684 # used) that's why we order ties by received_timestamp |
|
c13333fcde5e
memory (sqlite): fixed order for last_stanza_id:
Goffi <goffi@goffi.org>
parents:
2964
diff
changeset
|
685 # We'll reverse the order in sqliteHistoryToList |
|
c13333fcde5e
memory (sqlite): fixed order for last_stanza_id:
Goffi <goffi@goffi.org>
parents:
2964
diff
changeset
|
686 # we use DESC here so LIMIT keep the last messages |
| 3028 | 687 query_parts.append("ORDER BY timestamp DESC, history.received_timestamp DESC") |
|
1221
5e5661ab5c81
memory: getHistory with limit=0 now returns an empty list, use limit=None to return all messages
souliane <souliane@mailoo.org>
parents:
1099
diff
changeset
|
688 if limit is not None: |
| 3028 | 689 query_parts.append("LIMIT ?") |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
690 values.append(limit) |
|
587
952322b1d490
Remove trailing whitespaces.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
572
diff
changeset
|
691 |
| 3028 | 692 d = self.dbpool.runQuery(" ".join(query_parts), values) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
693 d.addCallback(self.sqliteHistoryToList) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
694 d.addCallback(self.listDict2listTuple) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
695 return d |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
696 |
| 2500 | 697 ## Private values |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
698 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
699 def _privateDataEb(self, failure_, operation, namespace, key=None, profile=None): |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
700 """generic errback for data queries""" |
| 3028 | 701 log.error(_("Can't {operation} data in database for namespace {namespace}{and_key}{for_profile}: {msg}").format( |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
702 operation = operation, |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
703 namespace = namespace, |
| 3028 | 704 and_key = (" and key " + key) if key is not None else "", |
| 705 for_profile = (' [' + profile + ']') if profile is not None else '', | |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
706 msg = failure_)) |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
707 |
| 3028 | 708 def _load_pickle(self, v): |
| 709 # FIXME: workaround for Python 3 port, some pickled data are byte while other are strings | |
| 710 try: | |
| 711 return pickle.loads(v) | |
| 712 except TypeError: | |
| 713 data = pickle.loads(v.encode('utf-8')) | |
| 714 log.warning(f"encoding issue in pickled data: {data}") | |
| 715 return data | |
| 716 | |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
717 def _generateDataDict(self, query_result, binary): |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
718 if binary: |
| 3028 | 719 return {k: self._load_pickle(v) for k,v in query_result} |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
720 else: |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
721 return dict(query_result) |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
722 |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
723 def _getPrivateTable(self, binary, profile): |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
724 """Get table to use for private values""" |
| 3028 | 725 table = ['private'] |
|
425
e4e9187e3b5b
backend, bridge: asynchronous history
Goffi <goffi@goffi.org>
parents:
423
diff
changeset
|
726 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
727 if profile is None: |
| 3028 | 728 table.append('gen') |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
729 else: |
| 3028 | 730 table.append('ind') |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
731 |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
732 if binary: |
| 3028 | 733 table.append('bin') |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
734 |
| 3028 | 735 return '_'.join(table) |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
736 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
737 def getPrivates(self, namespace, keys=None, binary=False, profile=None): |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
738 """Get private value(s) from databases |
|
592
e5a875a3311b
Fix pep8 support in src/memory.
Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
parents:
588
diff
changeset
|
739 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
740 @param namespace(unicode): namespace of the values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
741 @param keys(iterable, None): keys of the values to get |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
742 None to get all keys/values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
743 @param binary(bool): True to deserialise binary values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
744 @param profile(unicode, None): profile to use for individual values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
745 None to use general values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
746 @return (dict[unicode, object]): gotten keys/values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
747 """ |
| 3028 | 748 log.debug(_("getting {type}{binary} private values from database for namespace {namespace}{keys}".format( |
| 749 type = "general" if profile is None else "individual", | |
| 750 binary = " binary" if binary else "", | |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
751 namespace = namespace, |
| 3028 | 752 keys = " with keys {}".format(", ".join(keys)) if keys is not None else ""))) |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
753 table = self._getPrivateTable(binary, profile) |
| 3028 | 754 query_parts = ["SELECT key,value FROM", table, "WHERE namespace=?"] |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
755 args = [namespace] |
|
432
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
756 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
757 if keys is not None: |
| 3028 | 758 placeholders = ','.join(len(keys) * '?') |
| 759 query_parts.append('AND key IN (' + placeholders + ')') | |
|
2209
ea41cf1e6d29
memory (persistent, sqlite): fixed getPrivates bug with keys arguments and its use in LazyPersistentBinaryDict
Goffi <goffi@goffi.org>
parents:
2182
diff
changeset
|
760 args.extend(keys) |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
761 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
762 if profile is not None: |
| 3028 | 763 query_parts.append('AND profile_id=?') |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
764 args.append(self.profiles[profile]) |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
765 |
| 3028 | 766 d = self.dbpool.runQuery(" ".join(query_parts), args) |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
767 d.addCallback(self._generateDataDict, binary) |
| 3028 | 768 d.addErrback(self._privateDataEb, "get", namespace, profile=profile) |
|
432
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
769 return d |
|
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
770 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
771 def setPrivateValue(self, namespace, key, value, binary=False, profile=None): |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
772 """Set a private value in database |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
773 |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
774 @param namespace(unicode): namespace of the values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
775 @param key(unicode): key of the value to set |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
776 @param value(object): value to set |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
777 @param binary(bool): True if it's a binary values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
778 binary values need to be serialised, used for everything but strings |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
779 @param profile(unicode, None): profile to use for individual value |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
780 if None, it's a general value |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
781 """ |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
782 table = self._getPrivateTable(binary, profile) |
| 3028 | 783 query_values_names = ['namespace', 'key', 'value'] |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
784 query_values = [namespace, key] |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
785 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
786 if binary: |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
787 value = sqlite3.Binary(pickle.dumps(value, 0)) |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
788 |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
789 query_values.append(value) |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
790 |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
791 if profile is not None: |
| 3028 | 792 query_values_names.append('profile_id') |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
793 query_values.append(self.profiles[profile]) |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
794 |
| 3028 | 795 query_parts = ["REPLACE INTO", table, '(', ','.join(query_values_names), ')', |
| 796 "VALUES (", ",".join('?'*len(query_values_names)), ')'] | |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
797 |
| 3028 | 798 d = self.dbpool.runQuery(" ".join(query_parts), query_values) |
| 799 d.addErrback(self._privateDataEb, "set", namespace, key, profile=profile) | |
|
432
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
800 return d |
|
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
801 |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
802 def delPrivateValue(self, namespace, key, binary=False, profile=None): |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
803 """Delete private value from database |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
804 |
|
432
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
805 @param category: category of the privateeter |
|
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
806 @param key: key of the private value |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
807 @param binary(bool): True if it's a binary values |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
808 @param profile(unicode, None): profile to use for individual value |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
809 if None, it's a general value |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
810 """ |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
811 table = self._getPrivateTable(binary, profile) |
| 3028 | 812 query_parts = ["DELETE FROM", table, "WHERE namespace=? AND key=?"] |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
813 args = [namespace, key] |
|
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
814 if profile is not None: |
| 3028 | 815 query_parts.append("AND profile_id=?") |
|
2182
087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
Goffi <goffi@goffi.org>
parents:
2144
diff
changeset
|
816 args.append(self.profiles[profile]) |
| 3028 | 817 d = self.dbpool.runQuery(" ".join(query_parts), args) |
| 818 d.addErrback(self._privateDataEb, "delete", namespace, key, profile=profile) | |
|
432
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
819 return d |
|
31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
Goffi <goffi@goffi.org>
parents:
425
diff
changeset
|
820 |
|
2735
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
821 def delPrivateNamespace(self, namespace, binary=False, profile=None): |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
822 """Delete all data from a private namespace |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
823 |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
824 Be really cautious when you use this method, as all data with given namespace are |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
825 removed. |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
826 Params are the same as for delPrivateValue |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
827 """ |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
828 table = self._getPrivateTable(binary, profile) |
| 3028 | 829 query_parts = ["DELETE FROM", table, "WHERE namespace=?"] |
|
2735
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
830 args = [namespace] |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
831 if profile is not None: |
| 3028 | 832 query_parts.append("AND profile_id=?") |
|
2735
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
833 args.append(self.profiles[profile]) |
| 3028 | 834 d = self.dbpool.runQuery(" ".join(query_parts), args) |
| 835 d.addErrback(self._privateDataEb, "delete namespace", namespace, profile=profile) | |
|
2735
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
836 return d |
|
ba74914277cf
core (memory/sqlite): new delPrivateNamespace method to delete all data of a specific namespace for a profile.
Goffi <goffi@goffi.org>
parents:
2722
diff
changeset
|
837 |
| 2500 | 838 ## Files |
| 839 | |
| 840 @defer.inlineCallbacks | |
| 3028 | 841 def getFiles(self, client, file_id=None, version='', parent=None, type_=None, |
| 2500 | 842 file_hash=None, hash_algo=None, name=None, namespace=None, mime_type=None, |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
843 public_id=None, owner=None, access=None, projection=None, unique=False): |
| 2500 | 844 """retrieve files with with given filters |
| 845 | |
| 846 @param file_id(unicode, None): id of the file | |
| 847 None to ignore | |
| 848 @param version(unicode, None): version of the file | |
| 849 None to ignore | |
| 850 empty string to look for current version | |
| 851 @param parent(unicode, None): id of the directory containing the files | |
| 852 None to ignore | |
| 853 empty string to look for root files/directories | |
| 854 @param projection(list[unicode], None): name of columns to retrieve | |
| 855 None to retrieve all | |
| 856 @param unique(bool): if True will remove duplicates | |
| 857 other params are the same as for [setFile] | |
| 858 @return (list[dict]): files corresponding to filters | |
| 859 """ | |
| 860 query_parts = ["SELECT"] | |
| 861 if unique: | |
| 862 query_parts.append('DISTINCT') | |
| 863 if projection is None: | |
|
2507
4c45df43ea44
core (memory/sqlite): renamed column for files from hash to file_hash:
Goffi <goffi@goffi.org>
parents:
2500
diff
changeset
|
864 projection = ['id', 'version', 'parent', 'type', 'file_hash', 'hash_algo', 'name', |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
865 'size', 'namespace', 'media_type', 'media_subtype', 'public_id', 'created', 'modified', 'owner', |
| 2500 | 866 'access', 'extra'] |
| 867 query_parts.append(','.join(projection)) | |
| 868 query_parts.append("FROM files WHERE") | |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
869 if client is not None: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
870 filters = ['profile_id=?'] |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
871 args = [self.profiles[client.profile]] |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
872 else: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
873 if public_id is None: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
874 raise exceptions.InternalError( |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
875 "client can only be omitted when public_id is set") |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
876 filters = [] |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
877 args = [] |
| 2500 | 878 |
| 879 if file_id is not None: | |
| 3028 | 880 filters.append('id=?') |
| 2500 | 881 args.append(file_id) |
| 882 if version is not None: | |
| 3028 | 883 filters.append('version=?') |
| 2500 | 884 args.append(version) |
| 885 if parent is not None: | |
| 3028 | 886 filters.append('parent=?') |
| 2500 | 887 args.append(parent) |
| 888 if type_ is not None: | |
| 3028 | 889 filters.append('type=?') |
| 2500 | 890 args.append(type_) |
| 891 if file_hash is not None: | |
| 3028 | 892 filters.append('file_hash=?') |
| 2500 | 893 args.append(file_hash) |
| 894 if hash_algo is not None: | |
| 3028 | 895 filters.append('hash_algo=?') |
| 2500 | 896 args.append(hash_algo) |
| 897 if name is not None: | |
| 3028 | 898 filters.append('name=?') |
| 2500 | 899 args.append(name) |
| 900 if namespace is not None: | |
| 3028 | 901 filters.append('namespace=?') |
| 2500 | 902 args.append(namespace) |
| 903 if mime_type is not None: | |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
904 if '/' in mime_type: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
905 filters.extend('media_type=?', 'media_subtype=?') |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
906 args.extend(mime_type.split('/', 1)) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
907 else: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
908 filters.append('media_type=?') |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
909 args.append(mime_type) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
910 if public_id is not None: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
911 filters.append('public_id=?') |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
912 args.append(public_id) |
| 2500 | 913 if owner is not None: |
| 3028 | 914 filters.append('owner=?') |
| 2500 | 915 args.append(owner.full()) |
| 916 if access is not None: | |
| 917 raise NotImplementedError('Access check is not implemented yet') | |
| 918 # a JSON comparison is needed here | |
| 919 | |
| 3028 | 920 filters = ' AND '.join(filters) |
| 2500 | 921 query_parts.append(filters) |
| 3028 | 922 query = ' '.join(query_parts) |
| 2500 | 923 |
| 924 result = yield self.dbpool.runQuery(query, args) | |
| 3028 | 925 files_data = [dict(list(zip(projection, row))) for row in result] |
| 2500 | 926 to_parse = {'access', 'extra'}.intersection(projection) |
| 927 to_filter = {'owner'}.intersection(projection) | |
| 928 if to_parse or to_filter: | |
| 929 for file_data in files_data: | |
| 930 for key in to_parse: | |
| 931 value = file_data[key] | |
| 932 file_data[key] = {} if value is None else json.loads(value) | |
| 933 owner = file_data.get('owner') | |
| 934 if owner is not None: | |
| 935 file_data['owner'] = jid.JID(owner) | |
| 936 defer.returnValue(files_data) | |
| 937 | |
| 3028 | 938 def setFile(self, client, name, file_id, version='', parent=None, type_=C.FILE_TYPE_FILE, |
| 2500 | 939 file_hash=None, hash_algo=None, size=None, namespace=None, mime_type=None, |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
940 public_id=None, created=None, modified=None, owner=None, access=None, extra=None): |
| 2500 | 941 """set a file metadata |
| 942 | |
| 943 @param client(SatXMPPClient): client owning the file | |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
944 @param name(str): name of the file (must not contain "/") |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
945 @param file_id(str): unique id of the file |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
946 @param version(str): version of this file |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
947 @param parent(str): id of the directory containing this file |
| 2500 | 948 None if it is a root file/directory |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
949 @param type_(str): one of: |
| 2500 | 950 - file |
| 951 - directory | |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
952 @param file_hash(str): unique hash of the payload |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
953 @param hash_algo(str): algorithm used for hashing the file (usually sha-256) |
| 2500 | 954 @param size(int): size in bytes |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
955 @param namespace(str, None): identifier (human readable is better) to group files |
| 2500 | 956 for instance, namespace could be used to group files in a specific photo album |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
957 @param mime_type(str): media type of the file, or None if not known/guessed |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
958 @param public_id(str): ID used to server the file publicly via HTTP |
| 2500 | 959 @param created(int): UNIX time of creation |
| 960 @param modified(int,None): UNIX time of last modification, or None to use created date | |
| 961 @param owner(jid.JID, None): jid of the owner of the file (mainly useful for component) | |
| 962 @param access(dict, None): serialisable dictionary with access rules. See [memory.memory] for details | |
| 963 @param extra(dict, None): serialisable dictionary of any extra data | |
| 964 will be encoded to json in database | |
| 965 """ | |
| 966 if extra is not None: | |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
967 assert isinstance(extra, dict) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
968 |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
969 if mime_type is None: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
970 media_type = media_subtype = None |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
971 elif '/' in mime_type: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
972 media_type, media_subtype = mime_type.split('/', 1) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
973 else: |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
974 media_type, media_subtype = mime_type, None |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
975 |
|
2507
4c45df43ea44
core (memory/sqlite): renamed column for files from hash to file_hash:
Goffi <goffi@goffi.org>
parents:
2500
diff
changeset
|
976 query = ('INSERT INTO files(id, version, parent, type, file_hash, hash_algo, name, size, namespace, ' |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
977 'media_type, media_subtype, public_id, created, modified, owner, access, extra, profile_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)') |
| 2500 | 978 d = self.dbpool.runQuery(query, (file_id, version.strip(), parent, type_, |
| 979 file_hash, hash_algo, | |
| 980 name, size, namespace, | |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
981 media_type, media_subtype, public_id, created, modified, |
| 2500 | 982 owner.full() if owner is not None else None, |
| 983 json.dumps(access) if access else None, | |
| 984 json.dumps(extra) if extra else None, | |
| 985 self.profiles[client.profile])) | |
| 3028 | 986 d.addErrback(lambda failure: log.error(_("Can't save file metadata for [{profile}]: {reason}".format(profile=client.profile, reason=failure)))) |
| 2500 | 987 return d |
| 988 | |
|
2526
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
989 def _fileUpdate(self, cursor, file_id, column, update_cb): |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
990 query = 'SELECT {column} FROM files where id=?'.format(column=column) |
| 3028 | 991 for i in range(5): |
|
2526
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
992 cursor.execute(query, [file_id]) |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
993 try: |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
994 older_value_raw = cursor.fetchone()[0] |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
995 except TypeError: |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
996 raise exceptions.NotFound |
|
2918
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
997 if older_value_raw is None: |
|
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
998 value = {} |
|
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
999 else: |
|
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
1000 value = json.loads(older_value_raw) |
|
2526
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1001 update_cb(value) |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1002 value_raw = json.dumps(value) |
|
2918
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
1003 if older_value_raw is None: |
|
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
1004 update_query = 'UPDATE files SET {column}=? WHERE id=? AND {column} is NULL'.format(column=column) |
|
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
1005 update_args = (value_raw, file_id) |
|
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
1006 else: |
|
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
1007 update_query = 'UPDATE files SET {column}=? WHERE id=? AND {column}=?'.format(column=column) |
|
21cf8395616c
memory (sqlite): fixed fileUpdate when original value is not set (NULL)
Goffi <goffi@goffi.org>
parents:
2895
diff
changeset
|
1008 update_args = (value_raw, file_id, older_value_raw) |
|
2526
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1009 try: |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1010 cursor.execute(update_query, update_args) |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1011 except sqlite3.Error: |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1012 pass |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1013 else: |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1014 if cursor.rowcount == 1: |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1015 break; |
| 3028 | 1016 log.warning(_("table not updated, probably due to race condition, trying again ({tries})").format(tries=i+1)) |
|
2526
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1017 else: |
| 3028 | 1018 log.error(_("Can't update file table")) |
|
2526
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1019 |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1020 def fileUpdate(self, file_id, column, update_cb): |
|
2928
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1021 """Update a column value using a method to avoid race conditions |
|
2526
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1022 |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1023 the older value will be retrieved from database, then update_cb will be applied |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1024 to update it, and file will be updated checking that older value has not been changed meanwhile |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1025 by an other user. If it has changed, it tries again a couple of times before failing |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1026 @param column(str): column name (only "access" or "extra" are allowed) |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1027 @param update_cb(callable): method to update the value of the colum |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1028 the method will take older value as argument, and must update it in place |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1029 update_cb must not care about serialization, |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1030 it get the deserialized data (i.e. a Python object) directly |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1031 Note that the callable must be thread-safe |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1032 @raise exceptions.NotFound: there is not file with this id |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1033 """ |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1034 if column not in ('access', 'extra'): |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1035 raise exceptions.InternalError('bad column name') |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1036 return self.dbpool.runInteraction(self._fileUpdate, file_id, column, update_cb) |
|
35d591086974
core (memory, sqlite): added fileUpdate method to update "extra" and "access" avoiding race condition
Goffi <goffi@goffi.org>
parents:
2507
diff
changeset
|
1037 |
|
2928
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1038 def fileDelete(self, file_id): |
|
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1039 """Delete file metadata from the database |
|
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1040 |
|
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1041 @param file_id(unicode): id of the file to delete |
|
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1042 NOTE: file itself must still be removed, this method only handle metadata in |
|
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1043 database |
|
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1044 """ |
|
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1045 return self.dbpool.runQuery("DELETE FROM files WHERE id = ?", (file_id,)) |
|
c0f6fd75af5f
core (memory, memory/sqlite): implemented fileDelete
Goffi <goffi@goffi.org>
parents:
2919
diff
changeset
|
1046 |
|
413
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
1047 ##Helper methods## |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
1048 |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
1049 def __getFirstResult(self, result): |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
1050 """Return the first result of a database query |
|
dd4caab17008
core: - individual parameters managed through sqlite
Goffi <goffi@goffi.org>
parents:
412
diff
changeset
|
1051 Useful when we are looking for one specific value""" |
|
416
32dc8b18c2ae
core: param loading/purging on profile connection/disconnection
Goffi <goffi@goffi.org>
parents:
413
diff
changeset
|
1052 return None if not result else result[0][0] |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1053 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1054 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1055 class Updater(object): |
| 2500 | 1056 stmnt_regex = re.compile(r"[\w/' ]+(?:\(.*?\))?[^,]*") |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1057 clean_regex = re.compile(r"^ +|(?<= ) +|(?<=,) +| +$") |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1058 CREATE_SQL = "CREATE TABLE %s (%s)" |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1059 INSERT_SQL = "INSERT INTO %s VALUES (%s)" |
| 2787 | 1060 INDEX_SQL = "CREATE INDEX %s ON %s(%s)" |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1061 DROP_SQL = "DROP TABLE %s" |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1062 ALTER_SQL = "ALTER TABLE %s ADD COLUMN %s" |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1063 RENAME_TABLE_SQL = "ALTER TABLE %s RENAME TO %s" |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1064 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1065 CONSTRAINTS = ('PRIMARY', 'UNIQUE', 'CHECK', 'FOREIGN') |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1066 TMP_TABLE = "tmp_sat_update" |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1067 |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1068 def __init__(self, sqlite_storage, sat_version): |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1069 self._sat_version = sat_version |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1070 self.sqlite_storage = sqlite_storage |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1071 |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1072 @property |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1073 def dbpool(self): |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1074 return self.sqlite_storage.dbpool |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1075 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1076 def getLocalVersion(self): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1077 """ Get local database version |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
1078 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1079 @return: version (int) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1080 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1081 return self.dbpool.runQuery("PRAGMA user_version").addCallback(lambda ret: int(ret[0][0])) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1082 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1083 def _setLocalVersion(self, version): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1084 """ Set local database version |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
1085 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1086 @param version: version (int) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1087 @return: deferred |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1088 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1089 return self.dbpool.runOperation("PRAGMA user_version=%d" % version) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1090 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1091 def getLocalSchema(self): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1092 """ return raw local schema |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
1093 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1094 @return: list of strings with CREATE sql statements for local database |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1095 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1096 d = self.dbpool.runQuery("select sql from sqlite_master where type = 'table'") |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1097 d.addCallback(lambda result: [row[0] for row in result]) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1098 return d |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1099 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1100 @defer.inlineCallbacks |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1101 def checkUpdates(self): |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1102 """ Check if a database schema/content update is needed, according to DATABASE_SCHEMAS |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
1103 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1104 @return: deferred which fire a list of SQL update statements, or None if no update is needed |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1105 """ |
| 2787 | 1106 # TODO: only "table" type (i.e. "CREATE" statements) is checked, |
| 1107 # "index" should be checked too. | |
| 1108 # This may be not relevant is we move to a higher level library (alchimia?) | |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1109 local_version = yield self.getLocalVersion() |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1110 raw_local_sch = yield self.getLocalSchema() |
| 2500 | 1111 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1112 local_sch = self.rawStatements2data(raw_local_sch) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1113 current_sch = DATABASE_SCHEMAS['current']['CREATE'] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1114 local_hash = self.statementHash(local_sch) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1115 current_hash = self.statementHash(current_sch) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1116 |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1117 # Force the update if the schemas are unchanged but a specific update is needed |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1118 force_update = local_hash == current_hash and local_version < CURRENT_DB_VERSION \ |
| 2787 | 1119 and {'index', 'specific'}.intersection(DATABASE_SCHEMAS[CURRENT_DB_VERSION]) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1120 |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1121 if local_hash == current_hash and not force_update: |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1122 if local_version != CURRENT_DB_VERSION: |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
1123 log.warning(_("Your local schema is up-to-date, but database versions mismatch, fixing it...")) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1124 yield self._setLocalVersion(CURRENT_DB_VERSION) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1125 else: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1126 # an update is needed |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1127 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1128 if local_version == CURRENT_DB_VERSION: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1129 # Database mismatch and we have the latest version |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1130 if self._sat_version.endswith('D'): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1131 # we are in a development version |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1132 update_data = self.generateUpdateData(local_sch, current_sch, False) |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
1133 log.warning(_("There is a schema mismatch, but as we are on a dev version, database will be updated")) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1134 update_raw = yield self.update2raw(update_data, True) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1135 defer.returnValue(update_raw) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1136 else: |
| 3028 | 1137 log.error(_("schema version is up-to-date, but local schema differ from expected current schema")) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1138 update_data = self.generateUpdateData(local_sch, current_sch, True) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1139 update_raw = yield self.update2raw(update_data) |
| 3028 | 1140 log.warning(_("Here are the commands that should fix the situation, use at your own risk (do a backup before modifying database), you can go to SàT's MUC room at sat@chat.jabberfr.org for help\n### SQL###\n%s\n### END SQL ###\n") % '\n'.join("%s;" % statement for statement in update_raw)) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1141 raise exceptions.DatabaseError("Database mismatch") |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1142 else: |
|
2964
17c61d09a85b
memory (sqlite): exit with an error if current db version is higher than version expected in current code
Goffi <goffi@goffi.org>
parents:
2928
diff
changeset
|
1143 if local_version > CURRENT_DB_VERSION: |
|
17c61d09a85b
memory (sqlite): exit with an error if current db version is higher than version expected in current code
Goffi <goffi@goffi.org>
parents:
2928
diff
changeset
|
1144 log.error(_( |
| 3028 | 1145 "You database version is higher than the one used in this SàT " |
| 1146 "version, are you using several version at the same time? We " | |
| 1147 "can't run SàT with this database.")) | |
|
2964
17c61d09a85b
memory (sqlite): exit with an error if current db version is higher than version expected in current code
Goffi <goffi@goffi.org>
parents:
2928
diff
changeset
|
1148 sys.exit(1) |
|
17c61d09a85b
memory (sqlite): exit with an error if current db version is higher than version expected in current code
Goffi <goffi@goffi.org>
parents:
2928
diff
changeset
|
1149 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1150 # Database is not up-to-date, we'll do the update |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1151 if force_update: |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1152 log.info(_("Database content needs a specific processing, local database will be updated")) |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1153 else: |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1154 log.info(_("Database schema has changed, local database will be updated")) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1155 update_raw = [] |
| 3028 | 1156 for version in range(local_version + 1, CURRENT_DB_VERSION + 1): |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1157 try: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1158 update_data = DATABASE_SCHEMAS[version] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1159 except KeyError: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1160 raise exceptions.InternalError("Missing update definition (version %d)" % version) |
|
2720
453a12ff6f51
core (memory/sqlite): do not call commitStatements if there is nothing to commit in checkUpdates
Goffi <goffi@goffi.org>
parents:
2716
diff
changeset
|
1161 if "specific" in update_data and update_raw: |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1162 # if we have a specific, we must commit current statements |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1163 # because a specific may modify database itself, and the database |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1164 # must be in the expected state of the previous version. |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1165 yield self.sqlite_storage.commitStatements(update_raw) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1166 del update_raw[:] |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1167 update_raw_step = yield self.update2raw(update_data) |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1168 if update_raw_step is not None: |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1169 # can be None with specifics |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1170 update_raw.extend(update_raw_step) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1171 update_raw.append("PRAGMA user_version=%d" % CURRENT_DB_VERSION) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1172 defer.returnValue(update_raw) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1173 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1174 @staticmethod |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1175 def createData2Raw(data): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1176 """ Generate SQL statements from statements data |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
1177 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1178 @param data: dictionary with table as key, and statements data in tuples as value |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1179 @return: list of strings with raw statements |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1180 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1181 ret = [] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1182 for table in data: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1183 defs, constraints = data[table] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1184 assert isinstance(defs, tuple) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1185 assert isinstance(constraints, tuple) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1186 ret.append(Updater.CREATE_SQL % (table, ', '.join(defs + constraints))) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1187 return ret |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1188 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1189 @staticmethod |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1190 def insertData2Raw(data): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1191 """ Generate SQL statements from statements data |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
1192 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1193 @param data: dictionary with table as key, and statements data in tuples as value |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1194 @return: list of strings with raw statements |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1195 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1196 ret = [] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1197 for table in data: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1198 values_tuple = data[table] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1199 assert isinstance(values_tuple, tuple) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1200 for values in values_tuple: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1201 assert isinstance(values, tuple) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1202 ret.append(Updater.INSERT_SQL % (table, ', '.join(values))) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1203 return ret |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1204 |
| 2787 | 1205 @staticmethod |
| 1206 def indexData2Raw(data): | |
| 1207 """ Generate SQL statements from statements data | |
| 1208 | |
| 1209 @param data: dictionary with table as key, and statements data in tuples as value | |
| 1210 @return: list of strings with raw statements | |
| 1211 """ | |
| 1212 ret = [] | |
| 1213 assert isinstance(data, tuple) | |
| 1214 for table, col_data in data: | |
| 3028 | 1215 assert isinstance(table, str) |
| 2787 | 1216 assert isinstance(col_data, tuple) |
| 1217 for cols in col_data: | |
| 1218 if isinstance(cols, tuple): | |
| 3028 | 1219 assert all([isinstance(c, str) for c in cols]) |
| 1220 indexed_cols = ','.join(cols) | |
| 1221 elif isinstance(cols, str): | |
| 2787 | 1222 indexed_cols = cols |
| 1223 else: | |
| 3028 | 1224 raise exceptions.InternalError("unexpected index columns value") |
| 1225 index_name = table + '__' + indexed_cols.replace(',', '_') | |
| 2787 | 1226 ret.append(Updater.INDEX_SQL % (index_name, table, indexed_cols)) |
| 1227 return ret | |
| 1228 | |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1229 def statementHash(self, data): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1230 """ Generate hash of template data |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
1231 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1232 useful to compare schemas |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1233 @param data: dictionary of "CREATE" statement, with tables names as key, |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1234 and tuples of (col_defs, constraints) as values |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1235 @return: hash as string |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1236 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1237 hash_ = hashlib.sha1() |
| 3028 | 1238 tables = list(data.keys()) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1239 tables.sort() |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1240 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1241 def stmnts2str(stmts): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1242 return ','.join([self.clean_regex.sub('',stmt) for stmt in sorted(stmts)]) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1243 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1244 for table in tables: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1245 col_defs, col_constr = data[table] |
| 3028 | 1246 hash_.update( |
| 1247 ("%s:%s:%s" % (table, stmnts2str(col_defs), stmnts2str(col_constr))) | |
| 1248 .encode('utf-8')) | |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1249 return hash_.digest() |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1250 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1251 def rawStatements2data(self, raw_statements): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1252 """ separate "CREATE" statements into dictionary/tuples data |
|
2143
c3cac21157d4
memory (sqlite): introduced component table, schema updated:
Goffi <goffi@goffi.org>
parents:
2013
diff
changeset
|
1253 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1254 @param raw_statements: list of CREATE statements as strings |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1255 @return: dictionary with table names as key, and a (col_defs, constraints) tuple |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1256 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1257 schema_dict = {} |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1258 for create_statement in raw_statements: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1259 if not create_statement.startswith("CREATE TABLE "): |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
1260 log.warning("Unexpected statement, ignoring it") |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1261 continue |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1262 _create_statement = create_statement[13:] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1263 table, raw_col_stats = _create_statement.split(' ',1) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1264 if raw_col_stats[0] != '(' or raw_col_stats[-1] != ')': |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
1265 log.warning("Unexpected statement structure, ignoring it") |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1266 continue |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1267 col_stats = [stmt.strip() for stmt in self.stmnt_regex.findall(raw_col_stats[1:-1])] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1268 col_defs = [] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1269 constraints = [] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1270 for col_stat in col_stats: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1271 name = col_stat.split(' ',1)[0] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1272 if name in self.CONSTRAINTS: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1273 constraints.append(col_stat) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1274 else: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1275 col_defs.append(col_stat) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1276 schema_dict[table] = (tuple(col_defs), tuple(constraints)) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1277 return schema_dict |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1278 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1279 def generateUpdateData(self, old_data, new_data, modify=False): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1280 """ Generate data for automatic update between two schema data |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1281 |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1282 @param old_data: data of the former schema (which must be updated) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1283 @param new_data: data of the current schema |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1284 @param modify: if True, always use "cols modify" table, else try to ALTER tables |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1285 @return: update data, a dictionary with: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1286 - 'create': dictionary of tables to create |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1287 - 'delete': tuple of tables to delete |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1288 - 'cols create': dictionary of columns to create (table as key, tuple of columns to create as value) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1289 - 'cols delete': dictionary of columns to delete (table as key, tuple of columns to delete as value) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1290 - 'cols modify': dictionary of columns to modify (table as key, tuple of old columns to transfert as value). With this table, a new table will be created, and content from the old table will be transfered to it, only cols specified in the tuple will be transfered. |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1291 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1292 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1293 create_tables_data = {} |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1294 create_cols_data = {} |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1295 modify_cols_data = {} |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1296 delete_cols_data = {} |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1297 old_tables = set(old_data.keys()) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1298 new_tables = set(new_data.keys()) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1299 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1300 def getChanges(set_olds, set_news): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1301 to_create = set_news.difference(set_olds) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1302 to_delete = set_olds.difference(set_news) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1303 to_check = set_news.intersection(set_olds) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1304 return tuple(to_create), tuple(to_delete), tuple(to_check) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1305 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1306 tables_to_create, tables_to_delete, tables_to_check = getChanges(old_tables, new_tables) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1307 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1308 for table in tables_to_create: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1309 create_tables_data[table] = new_data[table] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1310 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1311 for table in tables_to_check: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1312 old_col_defs, old_constraints = old_data[table] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1313 new_col_defs, new_constraints = new_data[table] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1314 for obj in old_col_defs, old_constraints, new_col_defs, new_constraints: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1315 if not isinstance(obj, tuple): |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
1316 raise exceptions.InternalError("Columns definitions must be tuples") |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1317 defs_create, defs_delete, ignore = getChanges(set(old_col_defs), set(new_col_defs)) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1318 constraints_create, constraints_delete, ignore = getChanges(set(old_constraints), set(new_constraints)) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1319 created_col_names = set([name.split(' ',1)[0] for name in defs_create]) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1320 deleted_col_names = set([name.split(' ',1)[0] for name in defs_delete]) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1321 if (created_col_names.intersection(deleted_col_names or constraints_create or constraints_delete) or |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1322 (modify and (defs_create or constraints_create or defs_delete or constraints_delete))): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1323 # we have modified columns, we need to transfer table |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1324 # we determinate which columns are in both schema so we can transfer them |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1325 old_names = set([name.split(' ',1)[0] for name in old_col_defs]) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1326 new_names = set([name.split(' ',1)[0] for name in new_col_defs]) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1327 modify_cols_data[table] = tuple(old_names.intersection(new_names)); |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1328 else: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1329 if defs_create: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1330 create_cols_data[table] = (defs_create) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1331 if defs_delete or constraints_delete: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1332 delete_cols_data[table] = (defs_delete) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1333 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1334 return {'create': create_tables_data, |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1335 'delete': tables_to_delete, |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1336 'cols create': create_cols_data, |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1337 'cols delete': delete_cols_data, |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1338 'cols modify': modify_cols_data |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1339 } |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1340 |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1341 @defer.inlineCallbacks |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1342 def update2raw(self, update, dev_version=False): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1343 """ Transform update data to raw SQLite statements |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1344 |
|
1455
4fb3280c4568
tmp (wokkel.rsm): use of super instead of direct call in PubSubRequest
Goffi <goffi@goffi.org>
parents:
1409
diff
changeset
|
1345 @param update: update data as returned by generateUpdateData |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1346 @param dev_version: if True, update will be done in dev mode: no deletion will be done, instead a message will be shown. This prevent accidental lost of data while working on the code/database. |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1347 @return: list of string with SQL statements needed to update the base |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1348 """ |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1349 ret = self.createData2Raw(update.get('create', {})) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1350 drop = [] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1351 for table in update.get('delete', tuple()): |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1352 drop.append(self.DROP_SQL % table) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1353 if dev_version: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1354 if drop: |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
1355 log.info("Dev version, SQL NOT EXECUTED:\n--\n%s\n--\n" % "\n".join(drop)) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1356 else: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1357 ret.extend(drop) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1358 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1359 cols_create = update.get('cols create', {}) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1360 for table in cols_create: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1361 for col_def in cols_create[table]: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1362 ret.append(self.ALTER_SQL % (table, col_def)) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1363 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1364 cols_delete = update.get('cols delete', {}) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1365 for table in cols_delete: |
|
993
301b342c697a
core: use of the new core.log module:
Goffi <goffi@goffi.org>
parents:
936
diff
changeset
|
1366 log.info("Following columns in table [%s] are not needed anymore, but are kept for dev version: %s" % (table, ", ".join(cols_delete[table]))) |
|
853
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1367 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1368 cols_modify = update.get('cols modify', {}) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1369 for table in cols_modify: |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1370 ret.append(self.RENAME_TABLE_SQL % (table, self.TMP_TABLE)) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1371 main, extra = DATABASE_SCHEMAS['current']['CREATE'][table] |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1372 ret.append(self.CREATE_SQL % (table, ', '.join(main + extra))) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1373 common_cols = ', '.join(cols_modify[table]) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1374 ret.append("INSERT INTO %s (%s) SELECT %s FROM %s" % (table, common_cols, common_cols, self.TMP_TABLE)) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1375 ret.append(self.DROP_SQL % self.TMP_TABLE) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1376 |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1377 insert = update.get('insert', {}) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1378 ret.extend(self.insertData2Raw(insert)) |
|
c2f6ada7858f
core (sqlite): automatic database update:
Goffi <goffi@goffi.org>
parents:
839
diff
changeset
|
1379 |
| 2787 | 1380 index = update.get('index', tuple()) |
| 1381 ret.extend(self.indexData2Raw(index)) | |
| 1382 | |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1383 specific = update.get('specific', None) |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1384 if specific: |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1385 cmds = yield getattr(self, specific)() |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1386 ret.extend(cmds or []) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1387 defer.returnValue(ret) |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1388 |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1389 def update_v9(self): |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1390 """Update database from v8 to v9 |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1391 |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1392 (public_id on file with UNIQUE constraint, files indexes fix, media_type split) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1393 """ |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1394 # we have to do a specific update because we can't set UNIQUE constraint when adding a column |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1395 # (see https://sqlite.org/lang_altertable.html#alter_table_add_column) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1396 log.info("Database update to v9") |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1397 |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1398 create = { |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1399 'files': (("id TEXT NOT NULL", "public_id TEXT", "version TEXT NOT NULL", |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1400 "parent TEXT NOT NULL", |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1401 "type TEXT CHECK(type in ('{file}', '{directory}')) NOT NULL DEFAULT '{file}'".format( |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1402 file=C.FILE_TYPE_FILE, directory=C.FILE_TYPE_DIRECTORY), |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1403 "file_hash TEXT", "hash_algo TEXT", "name TEXT NOT NULL", "size INTEGER", |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1404 "namespace TEXT", "media_type TEXT", "media_subtype TEXT", |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1405 "created DATETIME NOT NULL", "modified DATETIME", |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1406 "owner TEXT", "access TEXT", "extra TEXT", "profile_id INTEGER"), |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1407 ("PRIMARY KEY (id, version)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE", |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1408 "UNIQUE (public_id)")), |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1409 |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1410 } |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1411 index = tuple({'files': (('profile_id', 'owner', 'media_type', 'media_subtype'), |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1412 ('profile_id', 'owner', 'parent'))}.items()) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1413 # XXX: Sqlite doc recommends to do the other way around (i.e. create new table, |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1414 # copy, drop old table then rename), but the RENAME would then add |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1415 # "IF NOT EXISTS" which breaks the (admittely fragile) schema comparison. |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1416 # TODO: rework sqlite update management, don't try to automatically detect |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1417 # update, the database version is now enough. |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1418 statements = ["ALTER TABLE files RENAME TO files_old"] |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1419 statements.extend(Updater.createData2Raw(create)) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1420 cols = ','.join([col_stmt.split()[0] for col_stmt in create['files'][0] if "public_id" not in col_stmt]) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1421 old_cols = cols[:] |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1422 # we need to split mime_type to the new media_type and media_subtype |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1423 old_cols = old_cols.replace( |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1424 'media_type,media_subtype', |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1425 "substr(mime_type, 0, instr(mime_type,'/')),substr(mime_type, instr(mime_type,'/')+1)" |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1426 ) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1427 statements.extend([ |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1428 f"INSERT INTO files({cols}) SELECT {old_cols} FROM files_old", |
|
3317
83f25da66bec
core (memory): files are now public by default except for those in root directory:
Goffi <goffi@goffi.org>
parents:
3288
diff
changeset
|
1429 "DROP TABLE files_old", |
|
3288
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1430 ]) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1431 statements.extend(Updater.indexData2Raw(index)) |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1432 return statements |
|
780fb8dd07ef
core (memory/sqlite): new database schema (v9):
Goffi <goffi@goffi.org>
parents:
3191
diff
changeset
|
1433 |
| 2787 | 1434 def update_v8(self): |
| 1435 """Update database from v7 to v8 (primary keys order changes + indexes)""" | |
| 3028 | 1436 log.info("Database update to v8") |
| 2787 | 1437 statements = ["PRAGMA foreign_keys = OFF"] |
| 1438 | |
| 1439 # here is a copy of create and index data, we can't use "current" table | |
| 1440 # because it may change in a future version, which would break the update | |
| 1441 # when doing v8 | |
| 1442 create = { | |
| 1443 'param_gen': ( | |
| 1444 ("category TEXT", "name TEXT", "value TEXT"), | |
| 1445 ("PRIMARY KEY (category, name)",)), | |
| 1446 'param_ind': ( | |
| 1447 ("category TEXT", "name TEXT", "profile_id INTEGER", "value TEXT"), | |
| 1448 ("PRIMARY KEY (profile_id, category, name)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE")), | |
| 1449 'private_ind': ( | |
| 1450 ("namespace TEXT", "key TEXT", "profile_id INTEGER", "value TEXT"), | |
| 1451 ("PRIMARY KEY (profile_id, namespace, key)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE")), | |
| 1452 'private_ind_bin': ( | |
| 1453 ("namespace TEXT", "key TEXT", "profile_id INTEGER", "value BLOB"), | |
| 1454 ("PRIMARY KEY (profile_id, namespace, key)", "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE")), | |
| 1455 } | |
| 1456 index = ( | |
| 1457 ('history', (('profile_id', 'timestamp'), | |
| 1458 ('profile_id', 'received_timestamp'))), | |
| 1459 ('message', ('history_uid',)), | |
| 1460 ('subject', ('history_uid',)), | |
| 1461 ('thread', ('history_uid',)), | |
| 1462 ('files', ('profile_id', 'mime_type', 'owner', 'parent'))) | |
| 1463 | |
| 1464 for table in ('param_gen', 'param_ind', 'private_ind', 'private_ind_bin'): | |
| 1465 statements.append("ALTER TABLE {0} RENAME TO {0}_old".format(table)) | |
| 1466 schema = {table: create[table]} | |
| 1467 cols = [d.split()[0] for d in schema[table][0]] | |
| 1468 statements.extend(Updater.createData2Raw(schema)) | |
| 3028 | 1469 statements.append("INSERT INTO {table}({cols}) " |
| 1470 "SELECT {cols} FROM {table}_old".format( | |
| 2787 | 1471 table=table, |
| 3028 | 1472 cols=','.join(cols))) |
| 1473 statements.append("DROP TABLE {}_old".format(table)) | |
| 2787 | 1474 |
| 1475 statements.extend(Updater.indexData2Raw(index)) | |
| 1476 statements.append("PRAGMA foreign_keys = ON") | |
| 1477 return statements | |
| 1478 | |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1479 @defer.inlineCallbacks |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1480 def update_v7(self): |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1481 """Update database from v6 to v7 (history unique constraint change)""" |
| 3028 | 1482 log.info("Database update to v7, this may be long depending on your history " |
| 1483 "size, please be patient.") | |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1484 |
| 3028 | 1485 log.info("Some cleaning first") |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1486 # we need to fix duplicate stanza_id, as it can result in conflicts with the new schema |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1487 # normally database should not contain any, but better safe than sorry. |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1488 rows = yield self.dbpool.runQuery( |
| 3028 | 1489 "SELECT stanza_id, COUNT(*) as c FROM history WHERE stanza_id is not NULL " |
| 1490 "GROUP BY stanza_id HAVING c>1") | |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1491 if rows: |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1492 count = sum([r[1] for r in rows]) - len(rows) |
| 3028 | 1493 log.info("{count} duplicate stanzas found, cleaning".format(count=count)) |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1494 for stanza_id, count in rows: |
| 3028 | 1495 log.info("cleaning duplicate stanza {stanza_id}".format(stanza_id=stanza_id)) |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1496 row_uids = yield self.dbpool.runQuery( |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1497 "SELECT uid FROM history WHERE stanza_id = ? LIMIT ?", |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1498 (stanza_id, count-1)) |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1499 uids = [r[0] for r in row_uids] |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1500 yield self.dbpool.runQuery( |
| 3028 | 1501 "DELETE FROM history WHERE uid IN ({})".format(",".join("?"*len(uids))), |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1502 uids) |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1503 |
|
2722
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1504 def deleteInfo(txn): |
|
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1505 # with foreign_keys on, the delete takes ages, so we deactivate it here |
|
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1506 # the time to delete info messages from history. |
|
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1507 txn.execute("PRAGMA foreign_keys = OFF") |
| 3028 | 1508 txn.execute("DELETE FROM message WHERE history_uid IN (SELECT uid FROM history WHERE " |
| 1509 "type='info')") | |
| 1510 txn.execute("DELETE FROM subject WHERE history_uid IN (SELECT uid FROM history WHERE " | |
| 1511 "type='info')") | |
| 1512 txn.execute("DELETE FROM thread WHERE history_uid IN (SELECT uid FROM history WHERE " | |
| 1513 "type='info')") | |
| 1514 txn.execute("DELETE FROM message WHERE history_uid IN (SELECT uid FROM history WHERE " | |
| 1515 "type='info')") | |
| 1516 txn.execute("DELETE FROM history WHERE type='info'") | |
|
2722
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1517 # not sure that is is necessary to reactivate here, but in doubt… |
|
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1518 txn.execute("PRAGMA foreign_keys = ON") |
|
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1519 |
| 3028 | 1520 log.info('Deleting "info" messages (this can take a while)') |
|
2722
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1521 yield self.dbpool.runInteraction(deleteInfo) |
|
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1522 |
| 3028 | 1523 log.info("Cleaning done") |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1524 |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1525 # we have to rename table we will replace |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1526 # tables referencing history need to be replaced to, else reference would |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1527 # be to the old table (which will be dropped at the end). This buggy behaviour |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1528 # seems to be fixed in new version of Sqlite |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1529 yield self.dbpool.runQuery("ALTER TABLE history RENAME TO history_old") |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1530 yield self.dbpool.runQuery("ALTER TABLE message RENAME TO message_old") |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1531 yield self.dbpool.runQuery("ALTER TABLE subject RENAME TO subject_old") |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1532 yield self.dbpool.runQuery("ALTER TABLE thread RENAME TO thread_old") |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1533 |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1534 # history |
| 3028 | 1535 query = ("CREATE TABLE history (uid TEXT PRIMARY KEY, stanza_id TEXT, " |
| 1536 "update_uid TEXT, profile_id INTEGER, source TEXT, dest TEXT, " | |
| 1537 "source_res TEXT, dest_res TEXT, timestamp DATETIME NOT NULL, " | |
| 1538 "received_timestamp DATETIME, type TEXT, extra BLOB, " | |
| 1539 "FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE, " | |
| 1540 "FOREIGN KEY(type) REFERENCES message_types(type), " | |
| 1541 "UNIQUE (profile_id, stanza_id, source, dest))") | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1542 yield self.dbpool.runQuery(query) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1543 |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1544 # message |
| 3028 | 1545 query = ("CREATE TABLE message (id INTEGER PRIMARY KEY ASC, history_uid INTEGER" |
| 1546 ", message TEXT, language TEXT, FOREIGN KEY(history_uid) REFERENCES " | |
| 1547 "history(uid) ON DELETE CASCADE)") | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1548 yield self.dbpool.runQuery(query) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1549 |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1550 # subject |
| 3028 | 1551 query = ("CREATE TABLE subject (id INTEGER PRIMARY KEY ASC, history_uid INTEGER" |
| 1552 ", subject TEXT, language TEXT, FOREIGN KEY(history_uid) REFERENCES " | |
| 1553 "history(uid) ON DELETE CASCADE)") | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1554 yield self.dbpool.runQuery(query) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1555 |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1556 # thread |
| 3028 | 1557 query = ("CREATE TABLE thread (id INTEGER PRIMARY KEY ASC, history_uid INTEGER" |
| 1558 ", thread_id TEXT, parent_id TEXT, FOREIGN KEY(history_uid) REFERENCES " | |
| 1559 "history(uid) ON DELETE CASCADE)") | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1560 yield self.dbpool.runQuery(query) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1561 |
| 3028 | 1562 log.info("Now transfering old data to new tables, please be patient.") |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1563 |
| 3028 | 1564 log.info("\nTransfering table history") |
| 1565 query = ("INSERT INTO history (uid, stanza_id, update_uid, profile_id, source, " | |
| 1566 "dest, source_res, dest_res, timestamp, received_timestamp, type, extra" | |
| 1567 ") SELECT uid, stanza_id, update_uid, profile_id, source, dest, " | |
| 1568 "source_res, dest_res, timestamp, received_timestamp, type, extra " | |
| 1569 "FROM history_old") | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1570 yield self.dbpool.runQuery(query) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1571 |
| 3028 | 1572 log.info("\nTransfering table message") |
| 1573 query = ("INSERT INTO message (id, history_uid, message, language) SELECT id, " | |
| 1574 "history_uid, message, language FROM message_old") | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1575 yield self.dbpool.runQuery(query) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1576 |
| 3028 | 1577 log.info("\nTransfering table subject") |
| 1578 query = ("INSERT INTO subject (id, history_uid, subject, language) SELECT id, " | |
| 1579 "history_uid, subject, language FROM subject_old") | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1580 yield self.dbpool.runQuery(query) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1581 |
| 3028 | 1582 log.info("\nTransfering table thread") |
| 1583 query = ("INSERT INTO thread (id, history_uid, thread_id, parent_id) SELECT id" | |
| 1584 ", history_uid, thread_id, parent_id FROM thread_old") | |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1585 yield self.dbpool.runQuery(query) |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1586 |
| 3028 | 1587 log.info("\nRemoving old tables") |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1588 # because of foreign keys, tables referencing history_old |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1589 # must be deleted first |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1590 yield self.dbpool.runQuery("DROP TABLE thread_old") |
|
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1591 yield self.dbpool.runQuery("DROP TABLE subject_old") |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1592 yield self.dbpool.runQuery("DROP TABLE message_old") |
|
2721
4aaa47f62d8d
core (memory/sqlite): fixed v7 update performance issue:
Goffi <goffi@goffi.org>
parents:
2720
diff
changeset
|
1593 yield self.dbpool.runQuery("DROP TABLE history_old") |
| 3028 | 1594 log.info("\nReducing database size (this can take a while)") |
|
2722
14e1db0c1383
core (memory/sqlite): further improvments in update_v7:
Goffi <goffi@goffi.org>
parents:
2721
diff
changeset
|
1595 yield self.dbpool.runQuery("VACUUM") |
| 3028 | 1596 log.info("Database update done :)") |
|
2716
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1597 |
|
06160b529da6
core (memory/sqlite): changed history constraint
Goffi <goffi@goffi.org>
parents:
2715
diff
changeset
|
1598 @defer.inlineCallbacks |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1599 def update_v3(self): |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1600 """Update database from v2 to v3 (message refactoring)""" |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1601 # XXX: this update do all the messages in one huge transaction |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1602 # this is really memory consuming, but was OK on a reasonably |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1603 # big database for tests. If issues are happening, we can cut it |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1604 # in smaller transactions using LIMIT and by deleting already updated |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1605 # messages |
| 3028 | 1606 log.info("Database update to v3, this may take a while") |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1607 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1608 # we need to fix duplicate timestamp, as it can result in conflicts with the new schema |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1609 rows = yield self.dbpool.runQuery("SELECT timestamp, COUNT(*) as c FROM history GROUP BY timestamp HAVING c>1") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1610 if rows: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1611 log.info("fixing duplicate timestamp") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1612 fixed = [] |
|
2765
378188abe941
misc: replaced all "dummy" by the more conventional and readable "__" ("_" being used for gettext)
Goffi <goffi@goffi.org>
parents:
2747
diff
changeset
|
1613 for timestamp, __ in rows: |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1614 ids_rows = yield self.dbpool.runQuery("SELECT id from history where timestamp=?", (timestamp,)) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1615 for idx, (id_,) in enumerate(ids_rows): |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1616 fixed.append(id_) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1617 yield self.dbpool.runQuery("UPDATE history SET timestamp=? WHERE id=?", (float(timestamp) + idx * 0.001, id_)) |
| 3028 | 1618 log.info("fixed messages with ids {}".format(', '.join([str(id_) for id_ in fixed]))) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1619 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1620 def historySchema(txn): |
| 3028 | 1621 log.info("History schema update") |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1622 txn.execute("ALTER TABLE history RENAME TO tmp_sat_update") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1623 txn.execute("CREATE TABLE history (uid TEXT PRIMARY KEY, update_uid TEXT, profile_id INTEGER, source TEXT, dest TEXT, source_res TEXT, dest_res TEXT, timestamp DATETIME NOT NULL, received_timestamp DATETIME, type TEXT, extra BLOB, FOREIGN KEY(profile_id) REFERENCES profiles(id) ON DELETE CASCADE, FOREIGN KEY(type) REFERENCES message_types(type), UNIQUE (profile_id, timestamp, source, dest, source_res, dest_res))") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1624 txn.execute("INSERT INTO history (uid, profile_id, source, dest, source_res, dest_res, timestamp, type, extra) SELECT id, profile_id, source, dest, source_res, dest_res, timestamp, type, extra FROM tmp_sat_update") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1625 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1626 yield self.dbpool.runInteraction(historySchema) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1627 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1628 def newTables(txn): |
| 3028 | 1629 log.info("Creating new tables") |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1630 txn.execute("CREATE TABLE message (id INTEGER PRIMARY KEY ASC, history_uid INTEGER, message TEXT, language TEXT, FOREIGN KEY(history_uid) REFERENCES history(uid) ON DELETE CASCADE)") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1631 txn.execute("CREATE TABLE thread (id INTEGER PRIMARY KEY ASC, history_uid INTEGER, thread_id TEXT, parent_id TEXT, FOREIGN KEY(history_uid) REFERENCES history(uid) ON DELETE CASCADE)") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1632 txn.execute("CREATE TABLE subject (id INTEGER PRIMARY KEY ASC, history_uid INTEGER, subject TEXT, language TEXT, FOREIGN KEY(history_uid) REFERENCES history(uid) ON DELETE CASCADE)") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1633 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1634 yield self.dbpool.runInteraction(newTables) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1635 |
| 3028 | 1636 log.info("inserting new message type") |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1637 yield self.dbpool.runQuery("INSERT INTO message_types VALUES (?)", ('info',)) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1638 |
| 3028 | 1639 log.info("messages update") |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1640 rows = yield self.dbpool.runQuery("SELECT id, timestamp, message, extra FROM tmp_sat_update") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1641 total = len(rows) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1642 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1643 def updateHistory(txn, queries): |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1644 for query, args in iter(queries): |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1645 txn.execute(query, args) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1646 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1647 queries = [] |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1648 for idx, row in enumerate(rows, 1): |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1649 if idx % 1000 == 0 or total - idx == 0: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1650 log.info("preparing message {}/{}".format(idx, total)) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1651 id_, timestamp, message, extra = row |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1652 try: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1653 extra = pickle.loads(str(extra or "")) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1654 except EOFError: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1655 extra = {} |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1656 except Exception: |
| 3028 | 1657 log.warning("Can't handle extra data for message id {}, ignoring it".format(id_)) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1658 extra = {} |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1659 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1660 queries.append(("INSERT INTO message(history_uid, message) VALUES (?,?)", (id_, message))) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1661 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1662 try: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1663 subject = extra.pop('subject') |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1664 except KeyError: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1665 pass |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1666 else: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1667 try: |
| 3028 | 1668 subject = subject |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1669 except UnicodeEncodeError: |
| 3028 | 1670 log.warning("Error while decoding subject, ignoring it") |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1671 del extra['subject'] |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1672 else: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1673 queries.append(("INSERT INTO subject(history_uid, subject) VALUES (?,?)", (id_, subject))) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1674 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1675 received_timestamp = extra.pop('timestamp', None) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1676 try: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1677 del extra['archive'] |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1678 except KeyError: |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1679 # archive was not used |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1680 pass |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1681 |
|
1962
a45235d8dc93
memory (sqlite): fixed handling of extra (pickled data) by using sqlite3.Binary
Goffi <goffi@goffi.org>
parents:
1961
diff
changeset
|
1682 queries.append(("UPDATE history SET received_timestamp=?,extra=? WHERE uid=?",(id_, received_timestamp, sqlite3.Binary(pickle.dumps(extra, 0))))) |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1683 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1684 yield self.dbpool.runInteraction(updateHistory, queries) |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1685 |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1686 log.info("Dropping temporary table") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1687 yield self.dbpool.runQuery("DROP TABLE tmp_sat_update") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1688 log.info("Database update finished :)") |
|
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1689 |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1690 def update2raw_v2(self): |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1691 """Update the database from v1 to v2 (add passwords encryptions): |
|
1955
633b5c21aefd
backend, frontend: messages refactoring (huge commit, not finished):
Goffi <goffi@goffi.org>
parents:
1934
diff
changeset
|
1692 |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1693 - the XMPP password value is re-used for the profile password (new parameter) |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1694 - the profile password is stored hashed |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1695 - the XMPP password is stored encrypted, with the profile password as key |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1696 - as there are no other stored passwords yet, it is enough, otherwise we |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1697 would need to encrypt the other passwords as it's done for XMPP password |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1698 """ |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1699 xmpp_pass_path = ('Connection', 'Password') |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1700 |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1701 def encrypt_values(values): |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1702 ret = [] |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1703 list_ = [] |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1704 |
|
1065
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1705 def prepare_queries(result, xmpp_password): |
|
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1706 try: |
|
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1707 id_ = result[0][0] |
|
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1708 except IndexError: |
| 3028 | 1709 log.error("Profile of id %d is referenced in 'param_ind' but it doesn't exist!" % profile_id) |
|
1099
1baa116501fa
memory (sqlite): fixes upgrade Deferred issue
souliane <souliane@mailoo.org>
parents:
1065
diff
changeset
|
1710 return defer.succeed(None) |
|
1065
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1711 |
|
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1712 sat_password = xmpp_password |
|
3160
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1713 sat_cipher = PasswordHasher.hash(sat_password) |
|
1065
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1714 personal_key = BlockCipher.getRandomKey(base64=True) |
|
3160
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1715 personal_cipher = BlockCipher.encrypt(sat_password, personal_key) |
|
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1716 xmpp_cipher = BlockCipher.encrypt(personal_key, xmpp_password) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1717 |
|
3160
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1718 ret.append("INSERT INTO param_ind(category,name,profile_id,value) VALUES ('%s','%s',%s,'%s')" % |
|
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1719 (C.PROFILE_PASS_PATH[0], C.PROFILE_PASS_PATH[1], id_, sat_cipher)) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1720 |
|
3160
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1721 ret.append("INSERT INTO private_ind(namespace,key,profile_id,value) VALUES ('%s','%s',%s,'%s')" % |
|
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1722 (C.MEMORY_CRYPTO_NAMESPACE, C.MEMORY_CRYPTO_KEY, id_, personal_cipher)) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1723 |
|
3160
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1724 ret.append("REPLACE INTO param_ind(category,name,profile_id,value) VALUES ('%s','%s',%s,'%s')" % |
|
330a5f1d9eea
core (memory/crypto): replaced `PyCrypto` by `cryptography`:
Goffi <goffi@goffi.org>
parents:
3137
diff
changeset
|
1725 (xmpp_pass_path[0], xmpp_pass_path[1], id_, xmpp_cipher)) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1726 |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1727 |
|
1065
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1728 for profile_id, xmpp_password in values: |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1729 d = self.dbpool.runQuery("SELECT id FROM profiles WHERE id=?", (profile_id,)) |
|
1065
9378c80e3408
memory (sqlite): fixes upgrade to database v2
souliane <souliane@mailoo.org>
parents:
1047
diff
changeset
|
1730 d.addCallback(prepare_queries, xmpp_password) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1731 list_.append(d) |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1732 |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1733 d_list = defer.DeferredList(list_) |
|
2765
378188abe941
misc: replaced all "dummy" by the more conventional and readable "__" ("_" being used for gettext)
Goffi <goffi@goffi.org>
parents:
2747
diff
changeset
|
1734 d_list.addCallback(lambda __: ret) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1735 return d_list |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1736 |
|
1045
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1737 def updateLiberviaConf(values): |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1738 try: |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1739 profile_id = values[0][0] |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1740 except IndexError: |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1741 return # no profile called "libervia" |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1742 |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1743 def cb(selected): |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1744 try: |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1745 password = selected[0][0] |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1746 except IndexError: |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1747 log.error("Libervia profile exists but no password is set! Update Libervia configuration will be skipped.") |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1748 return |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1749 fixConfigOption('libervia', 'passphrase', password, False) |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1750 d = self.dbpool.runQuery("SELECT value FROM param_ind WHERE category=? AND name=? AND profile_id=?", xmpp_pass_path + (profile_id,)) |
|
1047
5f7f913c05ac
memory: synchronize the upgrade to database v2 to be sure libervia's profile password is read to be written in the config file before its encryption
souliane <souliane@mailoo.org>
parents:
1045
diff
changeset
|
1751 return d.addCallback(cb) |
|
1045
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1752 |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1753 d = self.dbpool.runQuery("SELECT id FROM profiles WHERE name='libervia'") |
|
65fffdcb97f1
memory: auto-update configuration file with libervia's passphrase when migrating the database
souliane <souliane@mailoo.org>
parents:
1030
diff
changeset
|
1754 d.addCallback(updateLiberviaConf) |
|
2765
378188abe941
misc: replaced all "dummy" by the more conventional and readable "__" ("_" being used for gettext)
Goffi <goffi@goffi.org>
parents:
2747
diff
changeset
|
1755 d.addCallback(lambda __: self.dbpool.runQuery("SELECT profile_id,value FROM param_ind WHERE category=? AND name=?", xmpp_pass_path)) |
|
1030
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1756 d.addCallback(encrypt_values) |
|
15f43b54d697
core, memory, bridge: added profile password + password encryption:
souliane <souliane@mailoo.org>
parents:
1019
diff
changeset
|
1757 return d |