Mercurial > libervia-web

comparison src/server/pages.py @ 985:64826e69f365

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
pages: cache mechanism, first draft: a cache mechanism has been implemented to retrieve pages with a complexe rendering and/or calling expensive methods (e.g. network calls). For now it's is done only for Pubsub and with service profile (i.e. profile used when user is not logged in). When a LiberviaPage use cache, node is subscribed, and as long as no event is received (even can be item update, item retraction, or node deletion), the cached page is returned. This is a first draft, it is planed to handle in the future logged users (which can be tricky as we must not let (un)subscribed node if user is not willing to), multi-nodes pages (e.g.: item + comments) and cache for page not depending on pubsub (e.g. chat).
author Goffi <goffi@goffi.org>
date Sun, 19 Nov 2017 17:18:14 +0100
parents f0fc28b3bd1e
children 6daa59d44ee2
comparison
equal deleted inserted replaced
984:f0fc28b3bd1e 985:64826e69f365
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. 18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from twisted.web import server 19 from twisted.web import server
20 from twisted.web import resource as web_resource 20 from twisted.web import resource as web_resource
21 from twisted.web import util as web_util 21 from twisted.web import util as web_util
22 from twisted.internet import defer 22 from twisted.internet import defer
23 from twisted.words.protocols.jabber import jid
23 from twisted.python import failure 24 from twisted.python import failure
24 25
25 from sat.core.i18n import _ 26 from sat.core.i18n import _
26 from sat.core import exceptions 27 from sat.core import exceptions
27 from sat.tools.common import uri as common_uri 28 from sat.tools.common import uri as common_uri
32 from libervia.server.utils import quote 33 from libervia.server.utils import quote
33 import libervia 34 import libervia
34 35
35 import os.path 36 import os.path
36 import urllib 37 import urllib
38 import time
39
40
41 class Cache(object):
42
43 def __init__(self, rendered):
44 self._created = time.time()
45 self._last_access = self._created
46 self._rendered = rendered
47
48 @property
49 def created(self):
50 return self._created
51
52 @property
53 def last_access(self):
54 return self._last_access
55
56 @last_access.setter
57 def last_access(self, timestamp):
58 self._last_access = timestamp
59
60 @property
61 def rendered(self):
62 return self._rendered
37 63
38 64
39 class LiberviaPage(web_resource.Resource): 65 class LiberviaPage(web_resource.Resource):
40 isLeaf = True # we handle subpages ourself 66 isLeaf = True # we handle subpages ourself
41 named_pages = {} 67 named_pages = {}
42 uri_callbacks = {} 68 uri_callbacks = {}
43 pages_redirects = {} 69 pages_redirects = {}
70 cache = {}
71 # Set of tuples (service/node/sub_id) of nodes subscribed for caching
72 # sub_id can be empty string if not handled by service
73 cache_pubsub_sub = set()
44 74
45 def __init__(self, host, root_dir, url, name=None, redirect=None, access=None, parse_url=None, 75 def __init__(self, host, root_dir, url, name=None, redirect=None, access=None, parse_url=None,
46 prepare_render=None, render=None, template=None, on_data_post=None): 76 prepare_render=None, render=None, template=None, on_data_post=None):
47 """initiate LiberviaPages 77 """initiate LiberviaPages
48 78
123 else: 153 else:
124 if render is not None: 154 if render is not None:
125 log.error(_(u"render can't be used at the same time as template")) 155 log.error(_(u"render can't be used at the same time as template"))
126 if parse_url is not None and not callable(parse_url): 156 if parse_url is not None and not callable(parse_url):
127 log.error(_(u"parse_url must be a callable")) 157 log.error(_(u"parse_url must be a callable"))
158
159 # if not None, next rendering will be cached
160 # it must then contain a list of the the keys to use (without the page instance)
161 # e.g. [C.SERVICE_PROFILE, "pubsub", server@example.tld, pubsub_node] 
162 self._do_cache = None
163
164 def __unicode__(self):
165 return u'LiberviaPage {name} at {url}'.format(
166 name = self.name or u'<anonymous>',
167 url = self.url)
168
169 def __str__(self):
170 return self.__unicode__.encode('utf-8')
128 171
129 @classmethod 172 @classmethod
130 def importPages(cls, host, parent=None, path=None): 173 def importPages(cls, host, parent=None, path=None):
131 """Recursively import Libervia pages""" 174 """Recursively import Libervia pages"""
132 if path is None: 175 if path is None:
322 """ 365 """
323 pathElement = request.postpath.pop(0) 366 pathElement = request.postpath.pop(0)
324 request.prepath.append(pathElement) 367 request.prepath.append(pathElement)
325 return urllib.unquote(pathElement).decode('utf-8') 368 return urllib.unquote(pathElement).decode('utf-8')
326 369
370 def checkCacheSubscribeCb(self, sub_id, service, node):
371 self.cache_pubsub_sub.add((service, node, sub_id))
372
373 def checkCacheSubscribeEb(self, failure_, service, node):
374 log.warning(_(u"Can't subscribe to node: {msg}").format(msg=failure_))
375 # FIXME: cache must be marked as unusable here
376
377 def psNodeWatchAddEb(self, failure_, service, node):
378 log.warning(_(u"Can't add node watched: {msg}").format(msg=failure_))
379
380 def checkCache(self, request, cache_type, **kwargs):
381 """check if a page is in cache and return cached version if suitable
382
383 this method may perform extra operation to handle cache (e.g. subscribing to a
384 pubsub node)
385 @param request(server.Request): current HTTP request
386 @param cache_type(int): on of C.CACHE_* const.
387 @param **kwargs: args according to cache_type:
388 C.CACHE_PROFILE:
389 service: pubsub service
390 node: pubsub node
391 short: short name of feature (needed if node is empty)
392
393 """
394 if request.postpath:
395 # we are not on the final page, no need to go further
396 return
397 profile = self.getProfile(request) or C.SERVICE_PROFILE
398
399 if cache_type == C.CACHE_PUBSUB:
400 service, node = kwargs['service'], kwargs['node']
401 if not node:
402 try:
403 short = kwargs['short']
404 node = self.host.ns_map[short]
405 except KeyError:
406 log.warning(_(u"Can't use cache for empty node without namespace set, please ensure to set \"short\" and that it is registered"))
407 return
408 if profile != C.SERVICE_PROFILE:
409 # only service profile is cache for now
410 return
411 try:
412 cache = self.cache[profile][cache_type][service][node][self]
413 except KeyError:
414 # no cache yet, let's subscribe to the pubsub node
415 d1 = self.host.bridgeCall('psSubscribe', service.full(), node, {}, profile)
416 d1.addCallback(self.checkCacheSubscribeCb, service, node)
417 d1.addErrback(self.checkCacheSubscribeEb, service, node)
418 d2 = self.host.bridgeCall('psNodeWatchAdd', service.full(), node, profile)
419 d2.addErrback(self.psNodeWatchAddEb, service, node)
420 self._do_cache = [profile, cache_type, service, node]
421 # we don't return the Deferreds as it is not needed to wait for
422 # the subscription to continue with page rendering
423 return
424
425 else:
426 raise exceptions.InternalError(u'Unknown cache_type')
427 log.debug(u'using cache for {page}'.format(page=self))
428 cache.last_access = time.time()
429 request.write(cache.rendered)
430 request.finish()
431 raise failure.Failure(exceptions.CancelError(u'cache is used'))
432
433 @classmethod
434 def onNodeEvent(cls, host, service, node, event_type, items, profile):
435 """Invalidate cache for all pages linked to this node"""
436 try:
437 cache = cls.cache[profile][C.CACHE_PUBSUB][jid.JID(service)][node]
438 except KeyError:
439 log.info(_(u'Removing subscription for {service}/{node}: '
440 u'the page is not cached').format(service=service, node=node))
441 d1 = host.bridgeCall('psUnsubscribe', service, node, profile)
442 d1.addErrback(lambda failure_:
443 log.warning(_(u"Can't unsubscribe from {service}/{node}: {msg}").format(
444 service=service, node=node, msg=failure_)))
445 d2 = host.bridgeCall('psNodeWatchAdd', service, node, profile)
446 # TODO: check why the page is not in cache, remove subscription?
447 d2.addErrback(lambda failure_:
448 log.warning(_(u"Can't remove watch for {service}/{node}: {msg}").format(
449 service=service, node=node, msg=failure_)))
450 else:
451 cache.clear()
452
327 def HTTPRedirect(self, request, url): 453 def HTTPRedirect(self, request, url):
328 """redirect to an URL using HTTP redirection 454 """redirect to an URL using HTTP redirection
329 455
330 @param request(server.Request): current HTTP request 456 @param request(server.Request): current HTTP request
331 @param url(unicode): url to redirect to 457 @param url(unicode): url to redirect to
412 538
413 def writeData(self, data, request): 539 def writeData(self, data, request):
414 """write data to transport and finish the request""" 540 """write data to transport and finish the request"""
415 if data is None: 541 if data is None:
416 self.pageError(request) 542 self.pageError(request)
417 request.write(data.encode('utf-8')) 543 data_encoded = data.encode('utf-8')
544 request.write(data_encoded)
418 request.finish() 545 request.finish()
546 if self._do_cache is not None:
547 cache = reduce(lambda d, k: d.setdefault(k, {}), self._do_cache, self.cache)
548 cache[self] = Cache(data_encoded)
549 log.debug(_(u'{page} put in cache for [{profile}]').format(
550 page=self,
551 profile=self._do_cache[0]))
552 self._do_cache = None
419 553
420 def _subpagesHandler(self, dummy, request): 554 def _subpagesHandler(self, dummy, request):
421 """render subpage if suitable 555 """render subpage if suitable
422 556
423 this method checks if there is still an unmanaged part of the path 557 this method checks if there is still an unmanaged part of the path