Mercurial > libervia-backend
changeset 2357:fa43e285df1d
core (memory/sqlite): better stability:
- timeout increased to 15 s
- if a query fail, it's retried up to 5 times
- foreign key PRAGMA is executed on every new connection object (should fix non working foreign keys)
those are Q&D workarounds for issues with Sqlite, before switching to a higher level module to handle it correctly.
author | Goffi <goffi@goffi.org> |
---|---|
date | Thu, 21 Sep 2017 21:53:28 +0200 |
parents | 6084aa557425 |
children | 71b10dd7a13a |
files | src/memory/sqlite.py |
diffstat | 1 files changed, 23 insertions(+), 7 deletions(-) [+] |
line wrap: on
line diff
--- a/src/memory/sqlite.py Fri Sep 08 07:58:10 2017 +0200 +++ b/src/memory/sqlite.py Thu Sep 21 21:53:28 2017 +0200 @@ -106,10 +106,25 @@ class ConnectionPool(adbapi.ConnectionPool): # Workaround to avoid IntegrityError causing (i)pdb to be launched in debug mode def _runQuery(self, trans, *args, **kw): + retry = kw.pop('query_retry', 6) try: trans.execute(*args, **kw) except sqlite3.IntegrityError as e: raise failure.Failure(e) + except Exception as e: + #Â FIXME: in case of error, we retry a couple of times + # this is a workaround, we need to move to better + # Sqlite integration, probably with high level library + retry -= 1 + if retry == 0: + log.error(_(u'too many db tries, we abandon! Error message: {msg}').format( + msg = e)) + raise e + log.warning(_(u'exception while running query, retrying ({try_}): {msg})').format( + try_ = 6 - retry, + msg = e)) + kw['query_retry'] = retry + return self._runQuery(trans, *args, **kw) return trans.fetchall() @@ -130,13 +145,11 @@ dir_ = os.path.dirname(db_filename) if not os.path.exists(dir_): os.makedirs(dir_, 0700) - self.dbpool = ConnectionPool("sqlite3", db_filename, check_same_thread=False) - # init_defer is the initialisation deferred, initialisation is ok when all its callbacks have been done - # XXX: foreign_keys activation doesn't seem to work, probably because of the multi-threading - # All the requests that need to use this feature should be run with runInteraction instead, - # so you can set the PRAGMA as it is done in self.deleteProfile - init_defer = self.dbpool.runOperation("PRAGMA foreign_keys = ON").addErrback(lambda x: log.error(_("Can't activate foreign keys"))) + def foreignKeysOn(sqlite): + sqlite.execute('PRAGMA foreign_keys = ON') + + self.dbpool = ConnectionPool("sqlite3", db_filename, cp_openfun=foreignKeysOn, check_same_thread=False, timeout=15) def getNewBaseSql(): log.info(_("The database is new, creating the tables")) @@ -157,6 +170,10 @@ d = self.dbpool.runInteraction(self._updateDb, tuple(statements)) return d + # init_defer is the initialisation deferred, initialisation is ok when all its callbacks have been done + + init_defer = defer.succeed(None) + init_defer.addCallback(lambda ignore: getNewBaseSql() if new_base else getUpdateSql()) init_defer.addCallback(commitStatements) @@ -253,7 +270,6 @@ def delete(txn): profile_id = self.profiles.pop(name) - txn.execute("PRAGMA foreign_keys = ON") txn.execute("DELETE FROM profiles WHERE name = ?", (name,)) # FIXME: the following queries should be done by the ON DELETE CASCADE # but it seems they are not, so we explicitly do them by security