changeset 1969:5fbe09b9b568

merged main branch
author Goffi <goffi@goffi.org>
date Fri, 24 Jun 2016 22:41:28 +0200 (2016-06-24)
parents de6faf9be715 (current diff) eca59bc4e6c6 (diff)
children 200cd707a46d
files frontends/src/jp/cmd_blog.py frontends/src/jp/cmd_profile.py frontends/src/primitivus/constants.py frontends/src/primitivus/primitivus
diffstat 11 files changed, 320 insertions(+), 32 deletions(-) [+]
line wrap: on
line diff
--- a/CHANGELOG	Mon Jun 20 23:07:53 2016 +0200
+++ b/CHANGELOG	Fri Jun 24 22:41:28 2016 +0200
@@ -1,5 +1,86 @@
 All theses changelogs are not exhaustive, please check the mercurial repository for more details.
 
+v 0.6.1 (xx/xx/2016):
+    - minimum Twisted version is now 15.2.0
+    - removed pyfeed and xe dependencies
+    - added mutagen to recommended in README4PACKAGERS
+    - use of /usr/bin/env instead of /usr/bin/python in shebang
+    - core:
+          - connection and initialisation improvements
+          - leave the handling of delay elements to XEP-0203
+    - memory:
+          - fixes handling of jids_list parameter type
+          - fixed exception when setting an empty password
+          - disco: better handling of response failures
+    - tools:
+          - improved repository version detection
+          - moved data methods from common to common.data_format
+          - added common.regex module
+    - new plugins:
+          - blog_import: generic plugin to import blogs to SàT
+          - blog_import_dotclear: import dotclear blog
+          - blog_import_dokuwiki: import dokuwiki blog
+          - extra_pep: display messages from extra PEP services
+          - syntax_wiki_dotclear: convert dotclear wiki syntax from and to XHTML
+          - directory_subscription: temporary plugin moved from XEP-0055 implementation
+    - already existing plugins:
+          - account:
+                - auto add some roster contacts at profile creation
+                - parameter "admin_email" is now "email_admin_lists"
+                - email sending improvements (new authentication options)
+                - email the admins when a profile is created from an existing XMPP account
+          - ip: changed URL of GET_IP_PAGE for the new one on salut-a-toi.org
+          - XEP-0045: fixes commands feedback
+          - XEP-0060: MAM (XEP-0313) integration
+          - XEP-0166, XEP-0260: better handling of proxy error
+          - XEP-0277:
+                - improve comments handling
+                - uses new "extra pep" plugin
+                - Atom feed retrieval is left to Libervia
+                - fixes XHTML content encapsulation and handling of new lines
+          - XEP-0313: cleaning and improvements
+          - text_syntaxes: various improvements
+    - tmp/wokkel:
+            - updated behaviour and namespace to new urn:xmpp:mam:1
+            - several MAM and RSM improvements
+    - frontends:
+          - printing the history and notifications are left to quick_frontend
+          - restore printing the day change while displaying history
+          - fixes main item update and auto addition of a scheme to HTML links
+          - added ui_show_cb in actionManager (frontend can handle the XMLUI display)
+    - jp:
+          - new start method used by all commands
+          - new --output option to change the format of command output:
+                - possible outputs for now are "default", "json" and "json_raw" 
+          - new commands:
+                - blog: import, edit, preview
+                - roster: get, stats, purge
+          - command "message" moved to "message send"
+          - command "profile list" uses the new --output option
+          - added constants for exit codes
+    - primitivus:
+          - detect direct pasting in the message bar
+          - add bracketed paste mode
+    - libervia:
+          - browser and server sides:
+               - replaced isRegistered call by a more generic getSessionMetadata
+               - new option "allow_registration" to enable/disable new accounts registration
+          - browser side:
+                - improvee the popup message banner
+                - replaced old favicon and display favicon counter
+                - fixed handling of connection with external JID
+                - improved some regexps
+                - replace "re" module usage with pure javascript
+          - server:
+                - added mechanism for URL redirections
+                - renamed "ssl_certificate" to "tls_certificate"
+                - new options "tls_private_key", "tls_chain options", "base_url_ext"
+                - blog:
+                      - several improvements
+                      - implement tags/categories
+                      - removed max_items as we use RSM
+                      - fixed atom feed, add link and categories (tags) elements
+
 v 0.6.0 (02/12/2015):
     - modification of the social contract according to the General Assembly of August, 19th 2014
     - improved launching/stopping scripts:
--- a/README4PACKAGERS	Mon Jun 20 23:07:53 2016 +0200
+++ b/README4PACKAGERS	Fri Jun 24 22:41:28 2016 +0200
@@ -28,7 +28,7 @@
 XDG
 zope.interface
 
-Recommended: markdown, html2text, netifaces, miniupnp
+Recommended: Mutagen, markdown, html2text, netifaces, miniupnp
 
 --------------------------------
 Dependencies for the Jp frontend
--- a/frontends/src/jp/base.py	Mon Jun 20 23:07:53 2016 +0200
+++ b/frontends/src/jp/base.py	Fri Jun 24 22:41:28 2016 +0200
@@ -39,6 +39,7 @@
 from sat_frontends.jp.constants import Const as C
 import xml.etree.ElementTree as ET  # FIXME: used temporarily to manage XMLUI
 import shlex
+from collections import OrderedDict
 
 if sys.version_info < (2, 7, 3):
     # XXX: shlex.split only handle unicode since python 2.7.3
@@ -116,6 +117,11 @@
         self._progress_id = None # TODO: manage several progress ids
         self.quit_on_progress_end = True
 
+        # outputs
+        self._outputs = {}
+        for type_ in C.OUTPUT_TYPES:
+            self._outputs[type_] = OrderedDict()
+
     @property
     def version(self):
         return self.bridge.getVersion()
@@ -179,6 +185,9 @@
             else:
                 print msg
 
+    def output(self, type_, name, data):
+        self._outputs[type_][name]['callback'](data)
+
     def addOnQuitCallback(self, callback, *args, **kwargs):
         """Add a callback which will be called on quit command
 
@@ -191,6 +200,14 @@
         finally:
             callbacks_list.append((callback, args, kwargs))
 
+    def getOutputChoices(self, output_type):
+        """Return valid output filters for output_type
+
+        @param output_type: True for default,
+            else can be any registered type
+        """
+        return self._outputs[output_type].keys()
+
     def _make_parents(self):
         self.parents = {}
 
@@ -220,32 +237,45 @@
     def add_parser_options(self):
         self.parser.add_argument('--version', action='version', version=("%(name)s %(version)s %(copyleft)s" % {'name': PROG_NAME, 'version': self.version, 'copyleft': COPYLEFT}))
 
-    def import_commands(self):
-        """ Automaticaly import commands to jp
+    def register_output(self, type_, name, callback, description=""):
+        if type_ not in C.OUTPUT_TYPES:
+            log.error(u"Invalid output type {}".format(type_))
+            return
+        self._outputs[type_][name] = {'callback': callback,
+                                      'description': description
+                                     }
+
+    def import_plugins(self):
+        """Automaticaly import commands and outputs in jp
+
         looks from modules names cmd_*.py in jp path and import them
-
         """
         path = os.path.dirname(sat_frontends.jp.__file__)
-        modules = (os.path.splitext(module)[0] for module in map(os.path.basename, iglob(os.path.join(path, "cmd_*.py"))))
-        for module_name in modules:
-            module = import_module("sat_frontends.jp."+module_name)
-            try:
-                self.import_command_module(module)
-            except ImportError:
-                continue
+        # XXX: outputs must be imported before commands as they are used for arguments
+        for type_, pattern in ((C.PLUGIN_OUTPUT, 'output_*.py'), (C.PLUGIN_CMD, 'cmd_*.py')):
+            modules = (os.path.splitext(module)[0] for module in map(os.path.basename, iglob(os.path.join(path, pattern))))
+            for module_name in modules:
+                module = import_module("sat_frontends.jp."+module_name)
+                try:
+                    self.import_plugin_module(module, type_)
+                except ImportError:
+                    continue
 
-    def import_command_module(self, module):
-        """ Add commands from a module to jp
-        @param module: module containing commands
+    def import_plugin_module(self, module, type_):
+        """add commands or outpus from a module to jp
 
+        @param module: module containing commands or outputs
+        @param type_(str): one of C_PLUGIN_*
         """
         try:
-            for classname in module.__commands__:
-                cls = getattr(module, classname)
+            class_names =  getattr(module, '__{}__'.format(type_))
         except AttributeError:
-            log.warning(_("Invalid module %s") % module)
+            log.warning(_("Invalid plugin module [{type}] {module}").format(type=type_, module=module))
             raise ImportError
-        cls(self)
+        else:
+            for class_name in class_names:
+                cls = getattr(module, class_name)
+                cls(self)
 
     def run(self, args=None):
         self.args = self.parser.parse_args(args)
@@ -405,20 +435,23 @@
 
 class CommandBase(object):
 
-    def __init__(self, host, name, use_profile=True, use_progress=False, use_verbose=False, need_connect=None, help=None, **kwargs):
+    def __init__(self, host, name, use_profile=True, use_output=False,
+                       need_connect=None, help=None, **kwargs):
         """ Initialise CommandBase
         @param host: Jp instance
         @param name(unicode): name of the new command
         @param use_profile(bool): if True, add profile selection/connection commands
-        @param use_progress(bool): if True, add progress bar activation commands
-            progress* signals will be handled
-        @param use_verbose(bool): if True, add verbosity command
+        @param use_output(bool, dict): if not False, add --output option
         @param need_connect(bool, None): True if profile connection is needed
             False else (profile session must still be started)
             None to set auto value (i.e. True if use_profile is set)
             Can't be set if use_profile is False
         @param help(unicode): help message to display
         @param **kwargs: args passed to ArgumentParser
+            use_* are handled directly, they can be:
+            - use_progress(bool): if True, add progress bar activation option
+                progress* signals will be handled
+            - use_verbose(bool): if True, add verbosity option
         @attribute need_loop(bool): to set by commands when loop is needed
 
         """
@@ -428,6 +461,7 @@
         except AttributeError:
             self.host = host
 
+        # --profile option
         parents = kwargs.setdefault('parents', set())
         if use_profile:
             #self.host.parents['profile'] is an ArgumentParser with profile connection arguments
@@ -440,11 +474,29 @@
         # from this point, self.need_connect is None if connection is not needed at all
         # False if session starting is needed, and True if full connection is needed
 
-        if use_progress:
-            parents.add(self.host.parents['progress'])
+        # --output option
+        if use_output:
+            if use_output == True:
+                use_output = C.OUTPUT_TEXT
+            assert use_output in C.OUTPUT_TYPES
+            self._output_type = use_output
+            output_parent = argparse.ArgumentParser(add_help=False)
+            choices = self.host.getOutputChoices(use_output)
+            if not choices:
+                raise exceptions.InternalError("No choice found for {} output type".format(use_output))
+            default = 'default' if 'default' in choices else choices[0]
+            output_parent.add_argument('--output', '-O', choices=choices, default=default, help=_(u"Select output format"))
+            parents.add(output_parent)
 
-        if use_verbose:
-            parents.add(self.host.parents['verbose'])
+        # other common options
+        use_opts = {k:v for k,v in kwargs.iteritems() if k.startswith('use_')}
+        for param, do_use in use_opts.iteritems():
+            opt=param[4:] # if param is use_verbose, opt is verbose
+            if opt not in self.host.parents:
+                raise exceptions.InternalError(u"Unknown parent option {}".format(opt))
+            del kwargs[param]
+            if do_use:
+                parents.add(self.host.parents[opt])
 
         self.parser = host.subparsers.add_parser(name, help=help, **kwargs)
         if hasattr(self, "subcommands"):
@@ -572,6 +624,9 @@
     def disp(self, msg, verbosity=0, error=False):
         return self.host.disp(msg, verbosity, error)
 
+    def output(self, data):
+        return self.host.output(self._output_type, self.args.output, data)
+
     def add_parser_options(self):
         try:
             subcommands = self.subcommands
--- a/frontends/src/jp/cmd_blog.py	Mon Jun 20 23:07:53 2016 +0200
+++ b/frontends/src/jp/cmd_blog.py	Fri Jun 24 22:41:28 2016 +0200
@@ -147,7 +147,7 @@
         ext = os.path.splitext(path)[1][1:] # we get extension without the '.'
         if ext:
             for k,v in SYNTAX_EXT.iteritems():
-                if ext == v:
+                if k and ext == v:
                     return k
 
         # if not found, we use current syntax
--- a/frontends/src/jp/cmd_profile.py	Mon Jun 20 23:07:53 2016 +0200
+++ b/frontends/src/jp/cmd_profile.py	Fri Jun 24 22:41:28 2016 +0200
@@ -105,14 +105,13 @@
 
 class ProfileList(base.CommandBase):
     def __init__(self, host):
-        super(ProfileList, self).__init__(host, 'list', use_profile=False, help=_('list profiles'))
+        super(ProfileList, self).__init__(host, 'list', use_profile=False, use_output='list', help=_('list profiles'))
 
     def add_parser_options(self):
         pass
 
     def start(self):
-        for profile in self.host.bridge.getProfilesList():
-            print profile
+        self.output(self.host.bridge.getProfilesList())
 
 
 class ProfileCreate(base.CommandBase):
--- a/frontends/src/jp/constants.py	Mon Jun 20 23:07:53 2016 +0200
+++ b/frontends/src/jp/constants.py	Fri Jun 24 22:41:28 2016 +0200
@@ -23,3 +23,17 @@
 class Const(constants.Const):
 
     APP_NAME = "jp"
+    PLUGIN_CMD = "commands"
+    PLUGIN_OUTPUT = "outputs"
+    OUTPUT_TEXT = 'text' # blob of unicode text
+    OUTPUT_DICT = 'dict'
+    OUTPUT_LIST = 'list'
+    OUTPUT_TYPES = (OUTPUT_TEXT, OUTPUT_DICT, OUTPUT_LIST)
+
+    # exit codes
+    EXIT_OK = 0
+    EXIT_ERROR = 1 # generic error, when nothing else match
+    EXIT_BAD_ARG = 2 # arguments given by user are bad
+    EXIT_FILE_NOT_EXE = 126 # a file to be executed was found, but it was not an executable utility (cf. man 1 exit)
+    EXIT_CMD_NOT_FOUND = 127 # a utility to be executed was not found (cf. man 1 exit)
+    EXIT_SIGNAL_INT = 128 # a command was interrupted by a signal (cf. man 1 exit)
--- a/frontends/src/jp/jp	Mon Jun 20 23:07:53 2016 +0200
+++ b/frontends/src/jp/jp	Fri Jun 24 22:41:28 2016 +0200
@@ -21,5 +21,5 @@
 
 if __name__ == "__main__":
     jp = base.Jp()
-    jp.import_commands()
+    jp.import_plugins()
     jp.run()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/frontends/src/jp/output_std.py	Fri Jun 24 22:41:28 2016 +0200
@@ -0,0 +1,58 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+# jp: a SàT command line tool
+# Copyright (C) 2009-2016 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Standard outputs"""
+
+
+from sat_frontends.jp.constants import Const as C
+import json
+
+__outputs__ = ["Default", "Json"]
+DEFAULT = u'default'
+JSON = u'json'
+JSON_RAW = u'json_raw'
+
+
+class Default(object):
+    """Default outputs"""
+
+    def __init__(self, jp):
+        jp.register_output(C.OUTPUT_TEXT, DEFAULT, self.text)
+        jp.register_output(C.OUTPUT_LIST, DEFAULT, self.list)
+
+    def text(self, data):
+        print data
+
+    def list(self, data):
+        print u'\n'.join(data)
+
+
+class Json(object):
+    """outputs in json format"""
+
+    def __init__(self, jp):
+        jp.register_output(C.OUTPUT_LIST, JSON, self.dump_pretty)
+        jp.register_output(C.OUTPUT_LIST, JSON_RAW, self.dump)
+        jp.register_output(C.OUTPUT_DICT, JSON, self.dump_pretty)
+        jp.register_output(C.OUTPUT_DICT, JSON_RAW, self.dump)
+
+    def dump(self, data):
+        print json.dumps(data)
+
+    def dump_pretty(self, data):
+        print json.dumps(data, indent=4)
--- a/frontends/src/primitivus/constants.py	Mon Jun 20 23:07:53 2016 +0200
+++ b/frontends/src/primitivus/constants.py	Fri Jun 24 22:41:28 2016 +0200
@@ -23,6 +23,7 @@
 class Const(constants.Const):
 
     APP_NAME = "Primitivus"
+    SECTION_NAME = APP_NAME.lower()
     PALETTE = [
                ('title', 'black', 'light gray', 'standout,underline'),
                ('title_focus', 'white,bold', 'light gray', 'standout,underline'),
--- a/frontends/src/primitivus/primitivus	Mon Jun 20 23:07:53 2016 +0200
+++ b/frontends/src/primitivus/primitivus	Fri Jun 24 22:41:28 2016 +0200
@@ -24,7 +24,9 @@
 log_config.satConfigure(C.LOG_BACKEND_STANDARD, C)
 from sat.core import log as logging
 log = logging.getLogger(__name__)
+from sat.tools import config as sat_config
 import urwid
+from urwid.util import is_wide_char
 from urwid_satext import sat_widgets
 from urwid_satext.files_management import FileDialog
 from sat_frontends.bridge.DBus import DBusBridgeFrontend
@@ -43,6 +45,7 @@
 from sat_frontends.tools import jid
 from os.path import join
 import signal
+import sys
 
 
 class EditBar(sat_widgets.ModalEdit):
@@ -304,6 +307,12 @@
 
         # we already manage exit with a_key['APP_QUIT'], so we don't want C-c
         signal.signal(signal.SIGINT, signal.SIG_IGN)
+        sat_conf = sat_config.parseMainConf()
+        self._bracketed_paste = C.bool(sat_config.getConfig(sat_conf, C.SECTION_NAME, 'bracketed_paste', 'false'))
+        if self._bracketed_paste:
+            log.debug("setting bracketed paste mode as requested")
+            sys.stdout.write("\033[?2004h")
+            self._bracketed_mode_set = True
 
     @property
     def visible_widgets(self):
@@ -362,9 +371,69 @@
                 self.showPopUp(popup)
         super(PrimitivusApp, self).postInit(self.main_widget)
 
+    def keysToText(self, keys):
+        """Generator return normal text from urwid keys"""
+        for k in keys:
+            if k == 'tab':
+                yield u'\t'
+            elif k == 'enter':
+                yield u'\n'
+            elif is_wide_char(k,0) or (len(k)==1 and ord(k) >= 32):
+                yield k
+
     def inputFilter(self, input_, raw):
         if self.__saved_overlay and input_ != a_key['OVERLAY_HIDE']:
             return
+
+        ## paste detection/handling
+        if (len(input_) > 1 and                  # XXX: it may be needed to increase this value if buffer
+            not isinstance(input_[0], tuple) and #      or other things result in several chars at once
+            not 'window resize' in input_):      #      (e.g. using Primitivus through ssh). Need some testing
+                                                 #      and experience to adjust value.
+            if input_[0] == 'begin paste' and not self._bracketed_paste:
+                log.info(u"Bracketed paste mode detected")
+                self._bracketed_paste = True
+
+            if self._bracketed_paste:
+                # after this block, extra will contain non pasted keys
+                # and input_ will contain pasted keys
+                try:
+                    begin_idx = input_.index('begin paste')
+                except ValueError:
+                    # this is not a paste, maybe we have something buffering
+                    # or bracketed mode is set in conf but not enabled in term
+                    extra = input_
+                    input_ = []
+                else:
+                    try:
+                        end_idx = input_.index('end paste')
+                    except ValueError:
+                        log.warning(u"missing end paste sequence, discarding paste")
+                        extra = input_[:begin_idx]
+                        del input_[begin_idx:]
+                    else:
+                        extra = input_[:begin_idx] + input_[end_idx+1:]
+                        input_ = input_[begin_idx+1:end_idx]
+            else:
+                extra = None
+
+            log.debug(u"Paste detected (len {})".format(len(input_)))
+            try:
+                edit_bar = self.editBar
+            except AttributeError:
+                log.warning(u"Paste discarded: there is no edit bar yet")
+            else:
+                # XXX: if a paste is detected, we append it directly to the edit bar text
+                #      so the use can check it and press [enter] if it's OK
+                buf_paste = u''.join(self.keysToText(input_))
+                pos = edit_bar.edit_pos
+                edit_bar.set_edit_text(u'{}{}{}'.format(edit_bar.edit_text[:pos], buf_paste, edit_bar.edit_text[pos:]))
+                edit_bar.edit_pos+=len(buf_paste)
+            if not extra:
+                return
+            input_ = extra
+        ## end of paste detection/handling
+
         for i in input_:
             if isinstance(i,tuple):
                 if i[0] == 'mouse press':
@@ -776,6 +845,12 @@
 
     def onExitRequest(self, menu):
         QuickApp.onExit(self)
+        try:
+            if self._bracketed_mode_set: # we don't unset if bracketed paste mode was detected automatically (i.e. not in conf)
+                log.debug("unsetting bracketed paste mode")
+                sys.stdout.write("\033[?2004l")
+        except AttributeError:
+            pass
         raise urwid.ExitMainLoop()
 
     def onJoinRoomRequest(self, menu):
--- a/src/plugins/plugin_misc_ip.py	Mon Jun 20 23:07:53 2016 +0200
+++ b/src/plugins/plugin_misc_ip.py	Fri Jun 24 22:41:28 2016 +0200
@@ -23,6 +23,7 @@
 log = getLogger(__name__)
 from sat.tools import xml_tools
 from twisted.web import client as webclient
+from twisted.web import error as web_error
 from twisted.internet import defer
 from twisted.internet import reactor
 from twisted.internet import protocol
@@ -51,7 +52,8 @@
     "description": _("""This plugin help to discover our external IP address.""")
 }
 
-GET_IP_PAGE = "http://www.goffi.org/sat_tools/get_ip.php" # This page must only return external IP of the requester
+# TODO: GET_IP_PAGE should be configurable in sat.conf
+GET_IP_PAGE = "http://salut-a-toi.org/whereami/" # This page must only return external IP of the requester
 GET_IP_LABEL = D_(u"Allow external get IP")
 GET_IP_CATEGORY = "General"
 GET_IP_NAME = "allow_get_ip"
@@ -289,6 +291,9 @@
         except (internet_error.DNSLookupError, internet_error.TimeoutError):
             log.warning(u"Can't access Domain Name System")
             ip = None
+        except web_error.Error as e:
+            log.warning(u"Error while retrieving IP on {url}: {message}".format(url=GET_IP_PAGE, message=e))
+            ip = None
         else:
             self._external_ip_cache = ip
         defer.returnValue(ip)