# HG changeset patch # User Goffi # Date 1452260559 -3600 # Node ID 0271397635115a838e1d99e7adaecf67ea44e902 # Parent 2819e4241e78c65c0c047edf1318d09f8a2f31ce server (blog): cleaning & improvments: - use a constant for themes url - moved RSM related constants to server only constants, and renamed theme STATIC_RSM* - raised the default number of items/comments to 10 - removed references to microblog namespace as it is managed by backend - many little improvments for better readability - dont use dynamic relative paths anymore - replaced use of old formatting syntax (%) by format() - profile name in url is now properly (un)quoted - removed max_items as it was used at the same time as RSM (TODO: check RSM support before using it) - renamed render_* methods using camelCase for consistency - put a limit for rsm_max, to avoid overloading - don't sort items after getting them anymore, as sorting is already done by backend/pubsub according to request - use urllib.urlencode when possible diff -r 2819e4241e78 -r 027139763511 src/common/constants.py --- a/src/common/constants.py Fri Jan 08 14:29:52 2016 +0100 +++ b/src/common/constants.py Fri Jan 08 14:42:39 2016 +0100 @@ -17,7 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from sat.core.i18n import D_ from sat_frontends.quick_frontend import constants import os.path @@ -54,7 +53,3 @@ DEFAULT_AVATAR_URL = os.path.join(MEDIA_DIR, "misc", DEFAULT_AVATAR_FILE) EMPTY_AVATAR_FILE = "empty_avatar" EMPTY_AVATAR_URL = os.path.join(MEDIA_DIR, "misc", EMPTY_AVATAR_FILE) - - RSM_MAX_ITEMS = 5 - RSM_MAX_COMMENTS = 5 - diff -r 2819e4241e78 -r 027139763511 src/server/blog.py --- a/src/server/blog.py Fri Jan 08 14:29:52 2016 +0100 +++ b/src/server/blog.py Fri Jan 08 14:42:39 2016 +0100 @@ -30,17 +30,18 @@ from twisted.words.protocols.jabber.jid import JID from jinja2 import Environment, PackageLoader from datetime import datetime -from sys import path -import uuid import re import os +import sys +import urllib from libervia.server.html_tools import sanitizeHtml, convertNewLinesToXHTML from libervia.server.constants import Const as C -NS_MICROBLOG = 'urn:xmpp:microblog:0' +PARAMS_TO_GET = (C.STATIC_BLOG_PARAM_TITLE, C.STATIC_BLOG_PARAM_BANNER, C.STATIC_BLOG_PARAM_KEYWORDS, C.STATIC_BLOG_PARAM_DESCRIPTION) +# TODO: chech disco features and use max_items when RSM is not available class TemplateProcessor(object): @@ -50,13 +51,12 @@ self.host = host # add Libervia's themes directory to the python path - path.append(os.path.dirname(os.path.normpath(self.host.themes_dir))) + sys.path.append(os.path.dirname(os.path.normpath(self.host.themes_dir))) themes = os.path.basename(os.path.normpath(self.host.themes_dir)) self.env = Environment(loader=PackageLoader(themes, self.THEME)) def useTemplate(self, request, tpl, data=None): - root_url = '../' * len(request.postpath) - theme_url = os.path.join(root_url, 'themes', self.THEME) + theme_url = os.path.join('/', C.THEMES_URL, self.THEME) data_ = {'images': os.path.join(theme_url, 'images'), 'styles': os.path.join(theme_url, 'styles'), @@ -64,7 +64,7 @@ if data: data_.update(data) - template = self.env.get_template('%s.html' % tpl) + template = self.env.get_template('{}.html'.format(tpl)) return template.render(**data_).encode('utf-8') @@ -79,6 +79,22 @@ self.avatars_cache = {} self.waiting_deferreds = {} + def _quote(self, value): + """Quote a value for use in url + + @param value(unicode): value to quote + @return (str): quoted value + """ + return urllib.quote(value.encode('utf-8'), '') + + def _unquote(self, quoted_value): + """Unquote a value coming from url + + @param unquote_value(str): value to unquote + @return (unicode): unquoted value + """ + return urllib.unquote(quoted_value).decode('utf-8') + def entityDataUpdatedHandler(self, entity_s, key, value, dummy): """Retrieve the avatar we've been waiting for and fires the callback. @@ -94,8 +110,7 @@ url = os.path.join(C.AVATARS_DIR, value) self.avatars_cache[entity_s] = url try: - self.waiting_deferreds[entity_s].callback(url) - del self.waiting_deferreds[entity_s] + self.waiting_deferreds.pop(entity_s).callback(url) except KeyError: pass @@ -117,11 +132,11 @@ return defer.succeed(url if url else C.DEFAULT_AVATAR_URL) def render_GET(self, request): - if not request.postpath: + if not request.postpath or len(request.postpath) > 2: return self.useTemplate(request, "static_blog_error", {'message': "You must indicate a nickname"}) - prof_requested = request.postpath[0] - #TODO : char check: only use alphanumeric chars + some extra(_,-,...) here + prof_requested = self._unquote(request.postpath[0]) + try: prof_found = self.host.bridge.getProfileName(prof_requested) except DBusException: @@ -130,26 +145,31 @@ return self.useTemplate(request, "static_blog_error", {'message': "Invalid nickname"}) d = defer.Deferred() + # TODO: jid caching self.host.bridge.asyncGetParamA('JabberID', 'Connection', 'value', profile_key=prof_found, callback=d.callback, errback=d.errback) - d.addCallbacks(lambda pub_jid_s: self.gotJID(pub_jid_s, request, prof_found)) + d.addCallback(self.render_gotJID, request, prof_found) return server.NOT_DONE_YET - def gotJID(self, pub_jid_s, request, profile): + def render_gotJID(self, pub_jid_s, request, profile): pub_jid = JID(pub_jid_s) + request.extra_dict = {} # will be used for RSM and MAM self.parseURLParams(request) if request.item_id: + # we want a specific item item_ids = [request.item_id] max_items = 1 else: item_ids = [] - max_items = int(request.extra_dict['rsm_max']) + # max_items = int(request.extra_dict['rsm_max']) # FIXME + max_items = 0 + # TODO: use max_items only when RSM is not available if request.atom: - self.host.bridge.mbGetAtom(pub_jid.userhost(), NS_MICROBLOG, max_items, item_ids, + self.host.bridge.mbGetAtom(pub_jid.userhost(), '', max_items, item_ids, request.extra_dict, C.SERVICE_PROFILE, - lambda feed: self.render_atom_feed(feed, request), - lambda failure: self.render_error_blog(failure, request, pub_jid)) + lambda feed: self.renderAtomFeed(feed, request), + lambda failure: self.renderError(failure, request, pub_jid)) elif request.item_id: self.getItemById(pub_jid, request.item_id, request.extra_dict, request.extra_comments_dict, request, profile) @@ -157,36 +177,45 @@ self.getItems(pub_jid, max_items, request.extra_dict, request.extra_comments_dict, request, profile) + ## URL parsing + def parseURLParams(self, request): """Parse the request URL parameters. @param request: HTTP request """ - request.item_id = None - request.atom = False - if len(request.postpath) > 1: if request.postpath[1] == 'atom.xml': # return the atom feed request.atom = True + request.item_id = None else: + request.atom = False request.item_id = request.postpath[1] + else: + request.item_id = None + request.atom = False self.parseURLParamsRSM(request) + # XXX: request.display_single is True when only one blog post is visible request.display_single = (request.item_id is not None) or int(request.extra_dict['rsm_max']) == 1 self.parseURLParamsCommentsRSM(request) def parseURLParamsRSM(self, request): """Parse RSM request data from the URL parameters for main items + fill request.extra_dict accordingly @param request: HTTP request """ - request.extra_dict = {} if request.item_id: # XXX: item_id and RSM are not compatible return try: - request.extra_dict['rsm_max'] = request.args['max'][0] + rsm_max = int(request.args['max'][0]) + if rsm_max > C.STATIC_RSM_MAX_LIMIT: + log.warning(u"Request with rsm_max over limit ({})".format(rsm_max)) + rsm_max = C.STATIC_RSM_MAX_LIMIT + request.extra_dict['rsm_max'] = unicode(rsm_max) except (ValueError, KeyError): - request.extra_dict['rsm_max'] = unicode(C.RSM_MAX_ITEMS) + request.extra_dict['rsm_max'] = unicode(C.STATIC_RSM_MAX_DEFAULT) try: request.extra_dict['rsm_index'] = request.args['index'][0] except (ValueError, KeyError): @@ -201,17 +230,24 @@ def parseURLParamsCommentsRSM(self, request): """Parse RSM request data from the URL parameters for comments + fill request.extra_dict accordingly @param request: HTTP request """ request.extra_comments_dict = {} if request.display_single: try: - request.extra_comments_dict['rsm_max'] = request.args['comments_max'][0] + rsm_max = int(request.args['comments_max'][0]) + if rsm_max > C.STATIC_RSM_MAX_LIMIT: + log.warning(u"Request with rsm_max over limit ({})".format(rsm_max)) + rsm_max = C.STATIC_RSM_MAX_LIMIT + request.extra_comments_dict['rsm_max'] = unicode(rsm_max) except (ValueError, KeyError): - request.extra_comments_dict['rsm_max'] = unicode(C.RSM_MAX_COMMENTS) + request.extra_comments_dict['rsm_max'] = unicode(C.STATIC_RSM_MAX_COMMENTS_DEFAULT) else: request.extra_comments_dict['rsm_max'] = "0" + ## Items retrieval + def getItemById(self, pub_jid, item_id, extra_dict, extra_comments_dict, request, profile): """ @@ -227,30 +263,38 @@ items, metadata = items item = items[0] # assume there's only one item - def gotCount(items_bis): - metadata_bis = items_bis[1] - metadata['rsm_count'] = metadata_bis['rsm_count'] - index_key = "rsm_index" if metadata_bis.get("rsm_index") else "rsm_count" - metadata['rsm_index'] = unicode(int(metadata_bis[index_key]) - 1) + def gotMetadata(result): + dummy, rsm_metadata = result + try: + metadata['rsm_count'] = rsm_metadata['rsm_count'] + except KeyError: + pass + try: + metadata['rsm_index'] = unicode(int(rsm_metadata['rsm_index'])-1) + except KeyError: + pass + metadata['rsm_first'] = metadata['rsm_last'] = item["id"] def gotComments(comments): - # build the items as self.getItems would do it (and as self.render_html_blog expects them to be) + # build the items as self.getItems would do it (and as self.renderHTML expects them to be) comments = [(item['comments_service'], item['comments_node'], "", comments[0], comments[1])] - self.render_html_blog([(item, comments)], metadata, request, pub_jid, profile) + self.renderHTML([(item, comments)], metadata, request, pub_jid, profile) # get the comments - max_comments = int(extra_comments_dict['rsm_max']) + # max_comments = int(extra_comments_dict['rsm_max']) # FIXME + max_comments = 0 + # TODO: use max_comments only when RSM is not available self.host.bridge.mbGet(item['comments_service'], item['comments_node'], max_comments, [], extra_comments_dict, C.SERVICE_PROFILE, callback=gotComments) # XXX: retrieve RSM information related to the main item. We can't do it while # retrieving the item, because item_ids and rsm should not be used together. - self.host.bridge.mbGet(pub_jid.userhost(), NS_MICROBLOG, 1, [], - {"rsm_max": "1", "rsm_after": item["id"]}, C.SERVICE_PROFILE, callback=gotCount) + self.host.bridge.mbGet(pub_jid.userhost(), '', 0, [], + {"rsm_max": "1", "rsm_after": item["id"]}, C.SERVICE_PROFILE, callback=gotMetadata) # get the main item - self.host.bridge.mbGet(pub_jid.userhost(), NS_MICROBLOG, 1, [item_id], + self.host.bridge.mbGet(pub_jid.userhost(), '', 1, [item_id], extra_dict, C.SERVICE_PROFILE, callback=gotItems) def getItems(self, pub_jid, max_items, extra_dict, extra_comments_dict, request, profile): @@ -268,7 +312,7 @@ for result in results: service, node, failure, items, metadata = result if not failure: - self.render_html_blog(items, metadata, request, pub_jid, profile) + self.renderHTML(items, metadata, request, pub_jid, profile) if remaining: self._getResults(rt_session) @@ -276,17 +320,38 @@ def getResult(rt_session): self.host.bridge.mbGetFromManyWithCommentsRTResult(rt_session, C.SERVICE_PROFILE, callback=lambda data: getResultCb(data, rt_session), - errback=lambda failure: self.render_error_blog(failure, request, pub_jid)) + errback=lambda failure: self.renderError(failure, request, pub_jid)) - max_comments = int(extra_comments_dict['rsm_max']) + # max_comments = int(extra_comments_dict['rsm_max']) # FIXME + max_comments = 0 + # TODO: use max_comments only when RSM is not available self.host.bridge.mbGetFromManyWithComments(C.JID, [pub_jid.userhost()], max_items, max_comments, extra_dict, extra_comments_dict, C.SERVICE_PROFILE, callback=getResult) - def render_html_blog(self, items, metadata, request, pub_jid, profile): + ## rendering + + def _updateDict(self, value, dict_, key): + dict_[key] = value + + def _getImageParams(self, options, key, default, alt): + """regexp from http://answers.oreilly.com/topic/280-how-to-validate-urls-with-regular-expressions/""" + url = options[key] if key in options else '' + regexp = r"^(https?|ftp)://[a-z0-9-]+(\.[a-z0-9-]+)+(/[\w-]+)*/[\w-]+\.(gif|png|jpg)$" + if re.match(regexp, url): + url = url + else: + url = default + return BlogImage(url, alt) + + def renderError(self, failure, request, pub_jid): + request.write(self.useTemplate(request, "static_blog_error", {'message': "Can't access requested data"})) + request.finish() + + def renderHTML(self, items, metadata, request, pub_jid, profile): """Retrieve the user parameters before actually rendering the static blog - @param items(list[tuple(dict, list)]): same as in self.__render_html_blog + @param items(list[tuple(dict, list)]): same as in self._renderHTML @param metadata(dict): original node metadata @param request: HTTP request @param pub_jid (JID): publisher JID @@ -295,28 +360,27 @@ d_list = [] options = {} - def getCallback(param_name): - d = defer.Deferred() - d.addCallback(lambda value: options.update({param_name: value})) - d_list.append(d) - return d.callback - - eb = lambda failure: self.render_error_blog(failure, request, pub_jid) + d = self.getAvatarURL(pub_jid) + d.addCallback(self._updateDict, options, 'avatar') + d.addErrback(self.renderError, request, pub_jid) + d_list.append(d) - self.getAvatarURL(pub_jid).addCallbacks(getCallback('avatar'), eb) - for param_name in (C.STATIC_BLOG_PARAM_TITLE, C.STATIC_BLOG_PARAM_BANNER, C.STATIC_BLOG_PARAM_KEYWORDS, C.STATIC_BLOG_PARAM_DESCRIPTION): - self.host.bridge.asyncGetParamA(param_name, C.STATIC_BLOG_KEY, 'value', C.SERVER_SECURITY_LIMIT, profile, callback=getCallback(param_name), errback=eb) + for param_name in PARAMS_TO_GET: + d = defer.Deferred() + self.host.bridge.asyncGetParamA(param_name, C.STATIC_BLOG_KEY, 'value', C.SERVER_SECURITY_LIMIT, profile, callback=d.callback, errback=d.errback) + d.addCallback(self._updateDict, options, param_name) + d.addErrback(self.renderError, request, pub_jid) + d_list.append(d) - cb = lambda dummy: self.__render_html_blog(items, metadata, options, request, pub_jid) - defer.DeferredList(d_list).addCallback(cb) + dlist_d = defer.DeferredList(d_list) + dlist_d.addCallback(lambda dummy: self._renderHTML(items, metadata, options, request, pub_jid)) - def __render_html_blog(self, items, metadata, options, request, pub_jid): + def _renderHTML(self, items, metadata, options, request, pub_jid): """Actually render the static blog. If mblog_data is a list of dict, we are missing the comments items so we just display the main items. If mblog_data is a list of couple, each couple is associating a main item data with the list of its comments, so we render all. - @param items(list[tuple(dict, list)]): list of 2-tuple with - item(dict): item microblog data - comments_list(list[tuple]): list of 5-tuple with @@ -333,34 +397,21 @@ if not isinstance(options, dict): options = {} user = sanitizeHtml(pub_jid.user) - root_url = '../' * len(request.postpath) - base_url = root_url + 'blog/' + user + base_url = os.path.join('/blog/',user) def getOption(key): return sanitizeHtml(options[key]) if key in options else '' - def getImageParams(key, default, alt): - """regexp from http://answers.oreilly.com/topic/280-how-to-validate-urls-with-regular-expressions/""" - url = options[key] if key in options else '' - regexp = r"^(https?|ftp)://[a-z0-9-]+(\.[a-z0-9-]+)+(/[\w-]+)*/[\w-]+\.(gif|png|jpg)$" - if re.match(regexp, url): - url = url - else: - url = default - return BlogImage(url, alt) - - avatar = os.path.normpath(root_url + getOption('avatar')) + avatar = os.path.normpath('/{}'.format(getOption('avatar'))) title = getOption(C.STATIC_BLOG_PARAM_TITLE) or user data = {'base_url': base_url, 'keywords': getOption(C.STATIC_BLOG_PARAM_KEYWORDS), 'description': getOption(C.STATIC_BLOG_PARAM_DESCRIPTION), 'title': title, 'favicon': avatar, - 'banner_img': getImageParams(C.STATIC_BLOG_PARAM_BANNER, avatar, title) + 'banner_img': self._getImageParams(options, C.STATIC_BLOG_PARAM_BANNER, avatar, title) } - items.sort(key=lambda entry: (-float(entry[0].get('updated', 0)))) - data['navlinks'] = NavigationLinks(request, items, metadata, base_url) data['messages'] = [] for item in items: @@ -377,14 +428,10 @@ request.write(self.useTemplate(request, 'static_blog', data)) request.finish() - def render_atom_feed(self, feed, request): + def renderAtomFeed(self, feed, request): request.write(feed.encode('utf-8')) request.finish() - def render_error_blog(self, error, request, pub_jid): - request.write(self.useTemplate(request, "static_blog_error", {'message': "Can't access requested data"})) - request.finish() - class NavigationLinks(object): @@ -396,32 +443,54 @@ @param base_url (unicode): the base URL for this user's blog @return: dict """ - for key in ('later_message', 'later_messages', 'older_message', 'older_messages'): - count = int(rsm_data.get('rsm_count', 0)) - setattr(self, key, '') # key must exist when using the template - if count <= 0 or (request.display_single == key.endswith('s')): - continue + if request.display_single: + links = ('later_message', 'older_message') + # key must exist when using the template + self.later_messages = self.older_messages = '' + else: + links = ('later_messages', 'older_messages') + self.later_message = self.older_message = '' - index = int(rsm_data['rsm_index']) - - link_data = {'base_url': base_url, 'suffix': ''} + for key in links: + query_data = {} if key.startswith('later_message'): - if index <= 0: - continue - link_data['item_id'] = rsm_data['rsm_first'] - link_data['post_arg'] = 'before' + try: + index = int(rsm_data['rsm_index']) + except (KeyError, ValueError): + pass + else: + if index == 0: + # we don't show this link on first page + setattr(self, key, '') + continue + try: + query_data['before'] = rsm_data['rsm_first'].encode('utf-8') + except KeyError: + pass else: - if index + len(items) >= count: - continue - link_data['item_id'] = rsm_data['rsm_last'] - link_data['post_arg'] = 'after' + try: + index = int(rsm_data['rsm_index']) + count = int(rsm_data.get('rsm_count')) + except (KeyError, ValueError): + # XXX: if we don't have index or count, we can't know if we + # are on the last page or not + pass + else: + # if we have index, we don't show the after link + # on the last page + if index + len(items) >= count: + setattr(self, key, '') + continue + try: + query_data['after'] = rsm_data['rsm_last'].encode('utf-8') + except KeyError: + pass if request.display_single: - link_data['suffix'] = '&max=1' + query_data['max'] = 1 - link = "%(base_url)s?%(post_arg)s=%(item_id)s%(suffix)s" % link_data - + link = "{}?{}".format(base_url, urllib.urlencode(query_data)) setattr(self, key, BlogLink(link, key, key.replace('_', ' '))) @@ -462,32 +531,33 @@ self.content = self.getText(entry, 'content') if is_comment: - self.author = (_("from %s") % entry['author']) + self.author = (_("from {}").format(entry['author'])) else: self.author = ' ' - self.url = (u"%s/%s" % (base_url, entry['id'])) + self.url = "{}/{}".format(base_url, entry['id'].encode('utf-8')) self.title = self.getText(entry, 'title') self.tags = list(common.dict2iter('tag', entry)) count_text = lambda count: D_('comments') if count > 1 else D_('comment') - self.comments_text = "%s %s" % (comments_count, count_text(comments_count)) + self.comments_text = u"{} {}".format(comments_count, count_text(comments_count)) delta = comments_count - len(comments) if request.display_single and delta > 0: - prev_url = "%s?comments_max=%s" % (self.url, unicode(comments_count)) - prev_text = D_("show %(count)d previous %(comments)s") % \ - {'count': delta, 'comments': count_text(delta)} + prev_url = "{}?{}".format(self.url, urllib.urlencode({'comments_max', comments_count})) + prev_text = D_("show {count} previous {comments}").format( + count = delta, comments = count_text(delta)) self.all_comments_link = BlogLink(prev_url, "comments_link", prev_text) if comments: - comments.sort(key=lambda comment: float(comment.get('published', 0))) self.comments = [BlogMessage(request, base_url, comment) for comment in comments] def getText(self, entry, key): - if ('%s_xhtml' % key) in entry: - return entry['%s_xhtml' % key] - elif key in entry: - processor = addURLToText if key.startswith('content') else sanitizeHtml - return convertNewLinesToXHTML(processor(entry[key])) - return None + try: + return entry['{}_xhtml'.format(key)] + except KeyError: + try: + processor = addURLToText if key.startswith('content') else sanitizeHtml + return convertNewLinesToXHTML(processor(entry[key])) + except KeyError: + return None diff -r 2819e4241e78 -r 027139763511 src/server/constants.py --- a/src/server/constants.py Fri Jan 08 14:29:52 2016 +0100 +++ b/src/server/constants.py Fri Jan 08 14:42:39 2016 +0100 @@ -33,6 +33,7 @@ SESSION_TIMEOUT = 300 # Session's timeout, after that the user will be disconnected HTML_DIR = "html/" THEMES_DIR = "themes/" + THEMES_URL = "themes" MEDIA_DIR = "media/" CARDS_DIR = "games/cards/tarot" @@ -47,3 +48,7 @@ # keys for cache values we can get from browser ALLOWED_ENTITY_DATA = {'avatar', 'nick'} + + STATIC_RSM_MAX_LIMIT = 100 + STATIC_RSM_MAX_DEFAULT = 10 + STATIC_RSM_MAX_COMMENTS_DEFAULT = 10 diff -r 2819e4241e78 -r 027139763511 src/server/server.py --- a/src/server/server.py Fri Jan 08 14:29:52 2016 +0100 +++ b/src/server/server.py Fri Jan 08 14:42:39 2016 +0100 @@ -1281,6 +1281,7 @@ def putChild(path, resource): """Add a child to the root resource""" + # FIXME: check that no information is leaked (c.f. https://twistedmatrix.com/documents/current/web/howto/using-twistedweb.html#request-encoders) root.putChild(path, EncodingResourceWrapper(resource, [server.GzipEncoderFactory()])) putChild('', Redirect(C.LIBERVIA_MAIN_PAGE)) @@ -1291,7 +1292,7 @@ putChild('upload_radiocol', _upload_radiocol) putChild('upload_avatar', _upload_avatar) putChild('blog', MicroBlog(self)) - putChild('themes', ProtectedFile(self.themes_dir)) + putChild(C.THEMES_URL, ProtectedFile(self.themes_dir)) putChild(os.path.dirname(C.MEDIA_DIR), ProtectedFile(self.media_dir)) putChild(os.path.dirname(C.AVATARS_DIR), ProtectedFile(os.path.join(self.local_dir, C.AVATARS_DIR))) putChild('radiocol', ProtectedFile(_upload_radiocol.getTmpDir(), defaultType="audio/ogg")) # We cheat for PoC because we know we are on the same host, so we use directly upload dir