#!/usr/bin/python
#-*- coding: utf-8 -*-
#
# Copyright (c) 2012-2016 Jérôme Poisson
# Copyright (c) 2013-2016 Adrien Cossa
# Copyright (c) 2003-2011 Ralph Meijer


# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# --

# This program is based on Idavoll (http://idavoll.ik.nu/),
# originaly written by Ralph Meijer (http://ralphm.net/blog/)
# It is sublicensed under AGPL v3 (or any later version) as allowed by the original
# license.

# --

# Here is a copy of the original license:

# Copyright (c) 2003-2011 Ralph Meijer

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


"""
Generic publish-subscribe backend.

This module implements a generic publish-subscribe backend service with
business logic as per
U{XEP-0060<http://www.xmpp.org/extensions/xep-0060.html>} that interacts with
a given storage facility. It also provides an adapter from the XMPP
publish-subscribe protocol.
"""

import uuid

from zope.interface import implements

from twisted.application import service
from twisted.python import components, log
from twisted.internet import defer, reactor
from twisted.words.protocols.jabber.error import StanzaError
# from twisted.words.protocols.jabber.jid import JID, InvalidFormat
from twisted.words.xish import utility

from wokkel import disco
from wokkel import data_form
from wokkel import rsm
from wokkel import iwokkel
from wokkel import pubsub

from sat_pubsub import error
from sat_pubsub import iidavoll
from sat_pubsub import const
from sat_pubsub import container

from copy import deepcopy


def _getAffiliation(node, entity):
    d = node.getAffiliation(entity)
    d.addCallback(lambda affiliation: (node, affiliation))
    return d


class BackendService(service.Service, utility.EventDispatcher):
    """
    Generic publish-subscribe backend service.

    @cvar nodeOptions: Node configuration form as a mapping from the field
                       name to a dictionary that holds the field's type, label
                       and possible options to choose from.
    @type nodeOptions: C{dict}.
    @cvar defaultConfig: The default node configuration.
    """

    implements(iidavoll.IBackendService)

    nodeOptions = {
            const.OPT_PERSIST_ITEMS:
                {"type": "boolean",
                 "label": "Persist items to storage"},
            const.OPT_DELIVER_PAYLOADS:
                {"type": "boolean",
                 "label": "Deliver payloads with event notifications"},
            const.OPT_SEND_LAST_PUBLISHED_ITEM:
                {"type": "list-single",
                 "label": "When to send the last published item",
                 "options": {
                     "never": "Never",
                     "on_sub": "When a new subscription is processed"}
                },
            const.OPT_ACCESS_MODEL:
                {"type": "list-single",
                 "label": "Who can subscribe to this node",
                 "options": {
                     const.VAL_AMODEL_OPEN: "Public node",
                     const.VAL_AMODEL_ROSTER: "Node restricted to some roster groups",
                     const.VAL_AMODEL_JID: "Node restricted to some jids",
                     }
                },
            const.OPT_ROSTER_GROUPS_ALLOWED:
                {"type": "list-multi",
                 "label": "Groups of the roster allowed to access the node",
                },
            const.OPT_PUBLISH_MODEL:
                {"type": "list-single",
                 "label": "Who can publish to this node",
                 "options": {
                     const.VAL_PMODEL_OPEN: "Everybody can publish",
                     const.VAL_PMODEL_PUBLISHERS: "Only owner and publishers can publish",
                     const.VAL_PMODEL_SUBSCRIBERS: "Everybody which subscribed to the node",
                     }
                },
            }

    subscriptionOptions = {
            "pubsub#subscription_type":
                {"type": "list-single",
                 "options": {
                     "items": "Receive notification of new items only",
                     "nodes": "Receive notification of new nodes only"}
                },
            "pubsub#subscription_depth":
                {"type": "list-single",
                 "options": {
                     "1": "Receive notification from direct child nodes only",
                     "all": "Receive notification from all descendent nodes"}
                },
            }

    def __init__(self, storage):
        utility.EventDispatcher.__init__(self)
        self.storage = storage
        self._callbackList = []


    def supportsPublisherAffiliation(self):
        return True


    def supportsGroupBlog(self):
        return True


    def supportsOutcastAffiliation(self):
        return True


    def supportsPersistentItems(self):
        return True


    def supportsPublishModel(self):
        return True


    def getNodeType(self, nodeIdentifier, pep, recipient=None):
        # FIXME: manage pep and recipient
        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(lambda node: node.getType())
        return d


    def getNodes(self, pep):
        return self.storage.getNodeIds(pep)


    def getNodeMetaData(self, nodeIdentifier, pep, recipient=None):
        # FIXME: manage pep and recipient
        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(lambda node: node.getMetaData())
        d.addCallback(self._makeMetaData)
        return d


    def _makeMetaData(self, metaData):
        options = []
        for key, value in metaData.iteritems():
            if key in self.nodeOptions:
                option = {"var": key}
                option.update(self.nodeOptions[key])
                option["value"] = value
                options.append(option)

        return options


    def _checkAuth(self, node, requestor):
        """ Check authorisation of publishing in node for requestor """

        def check(affiliation):
            d = defer.succeed((affiliation, node))
            configuration = node.getConfiguration()
            publish_model = configuration[const.OPT_PUBLISH_MODEL]
            if publish_model == const.VAL_PMODEL_PUBLISHERS:
                if affiliation not in ['owner', 'publisher']:
                    raise error.Forbidden()
            elif publish_model == const.VAL_PMODEL_SUBSCRIBERS:
                if affiliation not in ['owner', 'publisher']:
                    # we are in subscribers publish model, we must check that
                    # the requestor is a subscriber to allow him to publish

                    def checkSubscription(subscribed):
                        if not subscribed:
                            raise error.Forbidden()
                        return (affiliation, node)

                    d.addCallback(lambda ignore: node.isSubscribed(requestor))
                    d.addCallback(checkSubscription)
            elif publish_model != const.VAL_PMODEL_OPEN:
                raise ValueError('Unexpected value') # publish_model must be publishers (default), subscribers or open.

            return d

        d = node.getAffiliation(requestor)
        d.addCallback(check)
        return d

    def parseItemConfig(self, item):
        """Get and remove item configuration information

        @param item (domish.Element): item to parse
        @return (tuple[unicode, dict)): (access_model, item_config)
        """
        item_config = None
        access_model = const.VAL_AMODEL_DEFAULT
        for idx, elt in enumerate(item.children):
            if elt.uri != 'data_form.NS_X_DATA' or elt.name != 'x':
                continue
            form = data_form.Form.fromElement(elt)
            if form.formNamespace == const.NS_ITEM_CONFIG:
                item_config = form
                del item.children[idx] #we need to remove the config from item
                break

        if item_config:
            access_model = item_config.get(const.OPT_ACCESS_MODEL, const.VAL_AMODEL_DEFAULT)
        return (access_model, item_config)

    def parseCategories(self, item_elt):
        """Check if item contain an atom entry, and parse categories if possible

        @param item (domish.Element): item to parse
        @return (list): list of found categories
        """
        categories = []
        try:
            entry_elt = item_elt.elements(const.NS_ATOM, "entry").next()
        except StopIteration:
            return categories

        for category_elt in entry_elt.elements(const.NS_ATOM, 'category'):
            category = category_elt.getAttribute('term')
            if category:
                categories.append(category)

        return categories

    def _checkOverwrite(self, node, itemIdentifiers, publisher):
        """Check that the itemIdentifiers correspond to items published
        by the current publisher"""
        def doCheck(item_pub_map):
            for item_publisher in item_pub_map.itervalues():
                if item_publisher.userhost() != publisher.userhost():
                    raise error.ItemForbidden()

        d = node.getItemsPublishers(itemIdentifiers)
        d.addCallback(doCheck)
        return d


    def publish(self, nodeIdentifier, items, requestor, pep, recipient):
        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(self._checkAuth, requestor)
        #FIXME: owner and publisher are not necessarly the same. So far we use only owner to get roster.
        #FIXME: in addition, there can be several owners: that is not managed yet
        d.addCallback(self._doPublish, items, requestor, pep, recipient)
        return d


    def _doPublish(self, result, items, requestor, pep, recipient):
        affiliation, node = result
        if node.nodeType == 'collection':
            raise error.NoPublishing()

        configuration = node.getConfiguration()
        persistItems = configuration[const.OPT_PERSIST_ITEMS]
        deliverPayloads = configuration[const.OPT_DELIVER_PAYLOADS]

        if items and not persistItems and not deliverPayloads:
            raise error.ItemForbidden()
        elif not items and (persistItems or deliverPayloads):
            raise error.ItemRequired()

        items_data = []
        check_overwrite = False
        for item in items:
            # we enforce publisher (cf XEP-0060 §7.1.2.3)
            item['publisher'] = requestor.full()
            if persistItems or deliverPayloads:
                item.uri = None
                item.defaultUri = None
                if not item.getAttribute("id"):
                    item["id"] = str(uuid.uuid4())
                else:
                    check_overwrite = True
            access_model, item_config = self.parseItemConfig(item)
            categories = self.parseCategories(item)
            items_data.append(container.ItemData(item, access_model, item_config, categories))

        if persistItems:
            if check_overwrite and affiliation != 'owner':
                # we don't want a publisher to overwrite the item
                # of an other publisher
                d = self._checkOverwrite(node, [item['id'] for item in items if item.getAttribute('id')], requestor)
                d.addCallback(lambda _: node.storeItems(items_data, requestor))
            else:
                d = node.storeItems(items_data, requestor)
        else:
            d = defer.succeed(None)

        d.addCallback(self._doNotify, node, items_data,
                      deliverPayloads, pep, recipient)
        return d


    def _doNotify(self, result, node, items_data, deliverPayloads, pep, recipient):
        if items_data and not deliverPayloads:
            for item_data in items_data:
                item_data.item.children = []
        self.dispatch({'items_data': items_data, 'node': node, 'pep': pep, 'recipient': recipient},
                      '//event/pubsub/notify')


    def getNotifications(self, nodeDbId, items_data):
        """Build a list of subscriber to the node

        subscribers will be associated with subscribed items,
        and subscription type.
        """

        def toNotifications(subscriptions, items_data):
            subsBySubscriber = {}
            for subscription in subscriptions:
                if subscription.options.get('pubsub#subscription_type',
                                            'items') == 'items':
                    subs = subsBySubscriber.setdefault(subscription.subscriber,
                                                       set())
                    subs.add(subscription)

            notifications = [(subscriber, subscriptions_, items_data)
                             for subscriber, subscriptions_
                             in subsBySubscriber.iteritems()]

            return notifications

        def rootNotFound(failure):
            failure.trap(error.NodeNotFound)
            return []

        d1 = self.storage.getNodeById(nodeDbId)
        d1.addCallback(lambda node: node.getSubscriptions('subscribed'))
        # FIXME: must add root node subscriptions ?
        # d2 = self.storage.getNode('', False) # FIXME: to check
        # d2.addCallback(lambda node: node.getSubscriptions('subscribed'))
        # d2.addErrback(rootNotFound)
        # d = defer.gatherResults([d1, d2])
        # d.addCallback(lambda result: result[0] + result[1])
        d1.addCallback(toNotifications, items_data)
        return d1

    def registerPublishNotifier(self, observerfn, *args, **kwargs):
        self.addObserver('//event/pubsub/notify', observerfn, *args, **kwargs)

    def registerRetractNotifier(self, observerfn, *args, **kwargs):
        self.addObserver('//event/pubsub/retract', observerfn, *args, **kwargs)

    def subscribe(self, nodeIdentifier, subscriber, requestor, pep, recipient):
        subscriberEntity = subscriber.userhostJID()
        if subscriberEntity != requestor.userhostJID():
            return defer.fail(error.Forbidden())

        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(_getAffiliation, subscriberEntity)
        d.addCallback(self._doSubscribe, subscriber)
        return d


    def _doSubscribe(self, result, subscriber):
        # TODO: implement other access models
        node, affiliation = result

        if affiliation == 'outcast':
            raise error.Forbidden()

        access_model = node.getAccessModel()

        if access_model != const.VAL_AMODEL_OPEN:
            raise NotImplementedError

        def trapExists(failure):
            failure.trap(error.SubscriptionExists)
            return False

        def cb(sendLast):
            d = node.getSubscription(subscriber)
            if sendLast:
                d.addCallback(self._sendLastPublished, node)
            return d

        d = node.addSubscription(subscriber, 'subscribed', {})
        d.addCallbacks(lambda _: True, trapExists)
        d.addCallback(cb)

        return d


    def _sendLastPublished(self, subscription, node):

        def notifyItem(items):
            if items:
                reactor.callLater(0, self.dispatch,
                                     {'items': items,
                                      'node': node,
                                      'subscription': subscription,
                                     },
                                     '//event/pubsub/notify')

        config = node.getConfiguration()
        sendLastPublished = config.get('pubsub#send_last_published_item',
                                       'never')
        if sendLastPublished == 'on_sub' and node.nodeType == 'leaf':
            entity = subscription.subscriber.userhostJID()
            d = self.getItems(node.nodeIdentifier, entity, 1)
            d.addCallback(notifyItem)
            d.addErrback(log.err)

        return subscription


    def unsubscribe(self, nodeIdentifier, subscriber, requestor, pep, recipient):
        if subscriber.userhostJID() != requestor.userhostJID():
            return defer.fail(error.Forbidden())

        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(lambda node: node.removeSubscription(subscriber))
        return d


    def getSubscriptions(self, entity):
        return self.storage.getSubscriptions(entity)

    def supportsAutoCreate(self):
        return True

    def supportsCreatorCheck(self):
        return True

    def supportsInstantNodes(self):
        return True


    def createNode(self, nodeIdentifier, requestor, options = None, pep=False, recipient=None):
        if not nodeIdentifier:
            nodeIdentifier = 'generic/%s' % uuid.uuid4()

        if not options:
            options = {}

        # if self.supportsCreatorCheck():
        #     groupblog = nodeIdentifier.startswith(const.NS_GROUPBLOG_PREFIX)
        #     try:
        #         nodeIdentifierJID = JID(nodeIdentifier[len(const.NS_GROUPBLOG_PREFIX):] if groupblog else nodeIdentifier)
        #     except InvalidFormat:
        #         is_user_jid = False
        #     else:
        #         is_user_jid = bool(nodeIdentifierJID.user)

        #     if is_user_jid and nodeIdentifierJID.userhostJID() != requestor.userhostJID():
        #         #we have an user jid node, but not created by the owner of this jid
        #         print "Wrong creator"
        #         raise error.Forbidden()

        nodeType = 'leaf'
        config = self.storage.getDefaultConfiguration(nodeType)
        config['pubsub#node_type'] = nodeType
        config.update(options)

        d = self.storage.createNode(nodeIdentifier, requestor, config, pep, recipient)
        d.addCallback(lambda _: nodeIdentifier)
        return d


    def getDefaultConfiguration(self, nodeType):
        d = defer.succeed(self.storage.getDefaultConfiguration(nodeType))
        return d


    def getNodeConfiguration(self, nodeIdentifier, pep, recipient):
        if not nodeIdentifier:
            return defer.fail(error.NoRootNode())

        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(lambda node: node.getConfiguration())

        return d


    def setNodeConfiguration(self, nodeIdentifier, options, requestor, pep, recipient):
        if not nodeIdentifier:
            return defer.fail(error.NoRootNode())

        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(_getAffiliation, requestor)
        d.addCallback(self._doSetNodeConfiguration, options)
        return d


    def _doSetNodeConfiguration(self, result, options):
        node, affiliation = result

        if affiliation != 'owner':
            raise error.Forbidden()

        return node.setConfiguration(options)


    def getAffiliations(self, entity):
        return self.storage.getAffiliations(entity)


    def getItems(self, nodeIdentifier, recipient, maxItems=None,
                       itemIdentifiers=None, ext_data=None):
        d = self.getItemsData(nodeIdentifier, recipient, maxItems, itemIdentifiers, ext_data)
        d.addCallback(lambda items_data: [item_data.item for item_data in items_data])
        return d

    def getItemsData(self, nodeIdentifier, recipient, maxItems=None,
                       itemIdentifiers=None, ext_data=None):
        """like getItems but return the whole ItemData"""
        if ext_data is None:
            ext_data = {}
        d = self.storage.getNode(nodeIdentifier, ext_data.get('pep', False), recipient)
        d.addCallback(_getAffiliation, recipient)
        d.addCallback(self._doGetItems, recipient, maxItems, itemIdentifiers,
                      ext_data)
        return d

    def checkGroup(self, roster_groups, entity):
        """Check that entity is authorized and in roster
        @param roster_group: tuple which 2 items:
                        - roster: mapping of jid to RosterItem as given by self.privilege.getRoster
                        - groups: list of authorized groups
        @param entity: entity which must be in group
        @return: (True, roster) if entity is in roster and authorized
                 (False, roster) if entity is in roster but not authorized
        @raise: error.NotInRoster if entity is not in roster"""
        roster, authorized_groups = roster_groups
        _entity = entity.userhostJID()

        if not _entity in roster:
            raise error.NotInRoster
        return (roster[_entity].groups.intersection(authorized_groups), roster)

    def _getNodeGroups(self, roster, nodeIdentifier, pep):
        d = self.storage.getNodeGroups(nodeIdentifier, pep)
        d.addCallback(lambda groups: (roster, groups))
        return d

    def _rosterEb(self, failure, owner_jid):
        log.msg("Error while getting roster of {}: {}".format(unicode(owner_jid), failure.value))
        return {}

    def _doGetItems(self, result, requestor, maxItems, itemIdentifiers,
                    ext_data):
        node, affiliation = result
        if maxItems == 0:
            log.msg("WARNING: maxItems=0 on items retrieval")
            return []

        def append_item_config(items_data):
            """Add item config data form to items with roster access model"""
            for item_data in items_data:
                if item_data.access_model == const.VAL_AMODEL_OPEN:
                    pass
                elif item_data.access_model == const.VAL_AMODEL_ROSTER:
                    form = data_form.Form('submit', formNamespace=const.NS_ITEM_CONFIG)
                    access = data_form.Field(None, const.OPT_ACCESS_MODEL, value=const.VAL_AMODEL_ROSTER)
                    allowed = data_form.Field(None, const.OPT_ROSTER_GROUPS_ALLOWED, values=item_data.config[const.OPT_ROSTER_GROUPS_ALLOWED])
                    form.addField(access)
                    form.addField(allowed)
                    item_data.item.addChild(form.toElement())
                elif access_model == const.VAL_AMODEL_JID:
                    #FIXME: manage jid
                    raise NotImplementedError
                else:
                    raise error.BadAccessTypeError(access_model)
            return items_data

        def access_checked(access_data):
            authorized, roster = access_data
            if not authorized:
                raise error.NotAuthorized()

            roster_item = roster.get(requestor.userhostJID())
            authorized_groups = tuple(roster_item.groups) if roster_item else tuple()
            owner = affiliation == 'owner'

            if itemIdentifiers:
                d = node.getItemsById(authorized_groups, owner, itemIdentifiers)
            else:
                d = node.getItems(authorized_groups, owner, maxItems, ext_data)
                if owner:
                    d.addCallback(append_item_config)

            d.addCallback(self._items_rsm, node, authorized_groups,
                          owner, itemIdentifiers,
                          ext_data)
            return d

        if not iidavoll.ILeafNode.providedBy(node):
            return []

        if affiliation == 'outcast':
            raise error.Forbidden()

        access_model = node.getAccessModel()
        d = node.getNodeOwner()

        def gotOwner(owner_jid):
            d_roster = self.privilege.getRoster(owner_jid)
            d_roster.addErrback(self._rosterEb, owner_jid)
            return d_roster

        d.addCallback(gotOwner)

        if access_model == const.VAL_AMODEL_OPEN or affiliation == 'owner':
            d.addCallback(lambda roster: (True, roster))
            d.addCallback(access_checked)
        elif access_model == const.VAL_AMODEL_ROSTER:
            d.addCallback(self._getNodeGroups, node.nodeIdentifier, ext_data.get('pep', False))
            d.addCallback(self.checkGroup, requestor)
            d.addCallback(access_checked)

        return d

    def _setCount(self, value, response):
        response.count = value

    def _setIndex(self, value, response, adjust):
        """Set index in RSM response

        @param value(int): value of the reference index (i.e. before or after item)
        @param response(RSMResponse): response instance to fill
        @param adjust(int): adjustement term (i.e. difference between reference index and first item of the result)
        """
        response.index = value + adjust

    def _items_rsm(self, items_data, node, authorized_groups, owner,
                   itemIdentifiers, ext_data):
        # FIXME: move this to a separate module
        # TODO: Index can be optimized by keeping a cache of the last RSM request
        #       An other optimisation would be to look for index first and use it as offset
        try:
            rsm_request = ext_data['rsm']
        except KeyError:
            # No RSM in this request, nothing to do
            return items_data

        if itemIdentifiers:
            log.msg("WARNING, itemIdentifiers used with RSM, ignoring the RSM part")
            return items_data

        response = rsm.RSMResponse()

        d_count = node.getItemsCount(authorized_groups, owner, ext_data)
        d_count.addCallback(self._setCount, response)
        d_list = [d_count]

        if items_data:
            response.first = items_data[0].item['id']
            response.last = items_data[-1].item['id']

            # index handling
            if rsm_request.index is not None:
                response.index = rsm_request.index
            elif rsm_request.before:
                # The last page case (before == '') is managed in render method
                d_index = node.getItemsIndex(rsm_request.before, authorized_groups, owner, ext_data)
                d_index.addCallback(self._setIndex, response, -len(items_data))
                d_list.append(d_index)
            elif rsm_request.after is not None:
                d_index = node.getItemsIndex(rsm_request.after, authorized_groups, owner, ext_data)
                d_index.addCallback(self._setIndex, response, 1)
                d_list.append(d_index)
            else:
                # the first page was requested
                response.index = 0

        def render(result):
            if rsm_request.before == '':
                # the last page was requested
                response.index = response.count - len(items_data)
            items_data.append(container.ItemData(response.toElement()))
            return items_data

        return defer.DeferredList(d_list).addCallback(render)

    def retractItem(self, nodeIdentifier, itemIdentifiers, requestor, notify, pep, recipient):
        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(_getAffiliation, requestor)
        d.addCallback(self._doRetract, itemIdentifiers, requestor, notify, pep, recipient)
        return d

    def _doRetract(self, result, itemIdentifiers, requestor, notify, pep, recipient):
        node, affiliation = result
        persistItems = node.getConfiguration()[const.OPT_PERSIST_ITEMS]

        if not persistItems:
            raise error.NodeNotPersistent()

        # we need to get the items before removing them, for the notifications

        def removeItems(items_data):
            """Remove the items and keep only actually removed ones in items_data"""
            d = node.removeItems(itemIdentifiers)
            d.addCallback(lambda removed: [item_data for item_data in items_data if item_data.item["id"] in removed])
            return d

        def checkPublishers(publishers_map):
            """Called when requestor is neither owner neither publisher of the Node

            We check that requestor is publisher of all the items he wants to retract
            and raise error.Forbidden if it is not the case
            """
            # TODO: the behaviour should be configurable (per node ?)
            if any((requestor.userhostJID() != publisher.userhostJID() for publisher in publishers_map.itervalues())):
                raise error.Forbidden()

        if affiliation in ['owner', 'publisher']:
            # the requestor is owner or publisher of the node
            # he can retract what he wants
            d = defer.succeed(None)
        else:
            # the requestor doesn't have right to retract on the whole node
            # we check if he is a publisher for all items he wants to retract
            # and forbid the retraction else.
            d = node.getItemsPublishers(itemIdentifiers)
            d.addCallback(checkPublishers)
        d.addCallback(lambda dummy: node.getItemsById(None, True, itemIdentifiers))
        d.addCallback(removeItems)

        if notify:
            d.addCallback(self._doNotifyRetraction, node, pep, recipient)
        return d


    def _doNotifyRetraction(self, items_data, node, pep, recipient):
        self.dispatch({'items_data': items_data,
                       'node': node,
                       'pep': pep,
                       'recipient': recipient},
                      '//event/pubsub/retract')


    def purgeNode(self, nodeIdentifier, requestor, pep, recipient):
        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(_getAffiliation, requestor)
        d.addCallback(self._doPurge)
        return d


    def _doPurge(self, result):
        node, affiliation = result
        persistItems = node.getConfiguration()[const.OPT_PERSIST_ITEMS]

        if affiliation != 'owner':
            raise error.Forbidden()

        if not persistItems:
            raise error.NodeNotPersistent()

        d = node.purge()
        d.addCallback(self._doNotifyPurge, node.nodeIdentifier)
        return d


    def _doNotifyPurge(self, result, nodeIdentifier):
        self.dispatch(nodeIdentifier, '//event/pubsub/purge')


    def registerPreDelete(self, preDeleteFn):
        self._callbackList.append(preDeleteFn)


    def getSubscribers(self, nodeIdentifier, pep, recipient):
        def cb(subscriptions):
            return [subscription.subscriber for subscription in subscriptions]

        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(lambda node: node.getSubscriptions('subscribed'))
        d.addCallback(cb)
        return d


    def deleteNode(self, nodeIdentifier, requestor, pep, recipient, redirectURI=None):
        d = self.storage.getNode(nodeIdentifier, pep, recipient)
        d.addCallback(_getAffiliation, requestor)
        d.addCallback(self._doPreDelete, redirectURI, pep, recipient)
        return d


    def _doPreDelete(self, result, redirectURI, pep, recipient):
        node, affiliation = result

        if affiliation != 'owner':
            raise error.Forbidden()

        data = {'node': node,
                'redirectURI': redirectURI}

        d = defer.DeferredList([cb(data, pep, recipient)
                                for cb in self._callbackList],
                               consumeErrors=1)
        d.addCallback(self._doDelete, node.nodeDbId)


    def _doDelete(self, result, nodeDbId):
        dl = []
        for succeeded, r in result:
            if succeeded and r:
                dl.extend(r)

        d = self.storage.deleteNodeByDbId(nodeDbId)
        d.addCallback(self._doNotifyDelete, dl)

        return d


    def _doNotifyDelete(self, result, dl):
        for d in dl:
            d.callback(None)



class PubSubResourceFromBackend(pubsub.PubSubResource):
    """
    Adapts a backend to an xmpp publish-subscribe service.
    """

    features = [
        "config-node",
        "create-nodes",
        "delete-any",
        "delete-nodes",
        "item-ids",
        "meta-data",
        "publish",
        "purge-nodes",
        "retract-items",
        "retrieve-affiliations",
        "retrieve-default",
        "retrieve-items",
        "retrieve-subscriptions",
        "subscribe",
    ]

    discoIdentity = disco.DiscoIdentity('pubsub',
                                        'service',
                                        u'Salut à Toi pubsub service')

    pubsubService = None

    _errorMap = {
        error.NodeNotFound: ('item-not-found', None, None),
        error.NodeExists: ('conflict', None, None),
        error.Forbidden: ('forbidden', None, None),
        error.NotAuthorized: ('not-authorized', None, None),
        error.NotInRoster: ('not-authorized', 'not-in-roster-group', None),
        error.ItemNotFound: ('item-not-found', None, None),
        error.ItemForbidden: ('bad-request', 'item-forbidden', None),
        error.ItemRequired: ('bad-request', 'item-required', None),
        error.NoInstantNodes: ('not-acceptable',
                               'unsupported',
                               'instant-nodes'),
        error.NotSubscribed: ('unexpected-request', 'not-subscribed', None),
        error.InvalidConfigurationOption: ('not-acceptable', None, None),
        error.InvalidConfigurationValue: ('not-acceptable', None, None),
        error.NodeNotPersistent: ('feature-not-implemented',
                                  'unsupported',
                                  'persistent-node'),
        error.NoRootNode: ('bad-request', None, None),
        error.NoCollections: ('feature-not-implemented',
                              'unsupported',
                              'collections'),
        error.NoPublishing: ('feature-not-implemented',
                             'unsupported',
                             'publish'),
    }

    def __init__(self, backend):
        pubsub.PubSubResource.__init__(self)

        self.backend = backend
        self.hideNodes = False

        self.backend.registerPublishNotifier(self._notifyPublish)
        self.backend.registerRetractNotifier(self._notifyRetract)
        self.backend.registerPreDelete(self._preDelete)

        # FIXME: to be removed, it's not useful anymore as PEP is now used
        # if self.backend.supportsCreatorCheck():
        #     self.features.append("creator-jid-check")  #SàT custom feature: Check that a node (which correspond to
                                                       #                    a jid in this server) is created by the right jid

        if self.backend.supportsAutoCreate():
            self.features.append("auto-create")

        if self.backend.supportsInstantNodes():
            self.features.append("instant-nodes")

        if self.backend.supportsOutcastAffiliation():
            self.features.append("outcast-affiliation")

        if self.backend.supportsPersistentItems():
            self.features.append("persistent-items")

        if self.backend.supportsPublisherAffiliation():
            self.features.append("publisher-affiliation")

        if self.backend.supportsGroupBlog():
            self.features.append("groupblog")

        # if self.backend.supportsPublishModel():       #XXX: this feature is not really described in XEP-0060, we just can see it in examples
        #     self.features.append("publish_model")     #     but it's necessary for microblogging comments (see XEP-0277)

    def _notifyPublish(self, data):
        items_data = data['items_data']
        node = data['node']
        pep = data['pep']
        recipient = data['recipient']

        def afterPrepare(result):
            owner_jid, notifications_filtered = result
            #we notify the owner
            #FIXME: check if this comply with XEP-0060 (option needed ?)
            #TODO: item's access model have to be sent back to owner
            #TODO: same thing for getItems

            def getFullItem(item_data):
                """ Attach item configuration to this item
                Used to give item configuration back to node's owner (and *only* to owner)
                """
                #TODO: a test should check that only the owner get the item configuration back

                item, item_config = item_data.item, item_data.config
                new_item = deepcopy(item)
                if item_config:
                    new_item.addChild(item_config.toElement())
                return new_item

            notifications_filtered.append((owner_jid,
                                           set([pubsub.Subscription(node.nodeIdentifier,
                                                            owner_jid,
                                                            'subscribed')]),
                                           [getFullItem(item_data) for item_data in items_data]))

            if pep:
                return self.backend.privilege.notifyPublish(
                    recipient,
                    node.nodeIdentifier,
                    notifications_filtered)

            else:
                return self.pubsubService.notifyPublish(
                    self.serviceJID,
                    node.nodeIdentifier,
                    notifications_filtered)

        d = self._prepareNotify(items_data, node, data.get('subscription'))
        d.addCallback(afterPrepare)
        return d

    def _notifyRetract(self, data):
        items_data = data['items_data']
        node = data['node']
        pep = data['pep']
        recipient = data['recipient']

        def afterPrepare(result):
            owner_jid, notifications_filtered = result
            #we add the owner

            notifications_filtered.append((owner_jid,
                                           set([pubsub.Subscription(node.nodeIdentifier,
                                                            owner_jid,
                                                            'subscribed')]),
                                           [item_data.item for item_data in items_data]))

            if pep:
                return self.backend.privilege.notifyRetract(
                    recipient,
                    node.nodeIdentifier,
                    notifications_filtered)

            else:
                return self.pubsubService.notifyRetract(
                    self.serviceJID,
                    node.nodeIdentifier,
                    notifications_filtered)

        d = self._prepareNotify(items_data, node, data.get('subscription'))
        d.addCallback(afterPrepare)
        return d


    def _prepareNotify(self, items_data, node, subscription=None):
        """Do a bunch of permissions check and filter notifications

        The owner is not added to these notifications,
        it must be called by the calling method
        @param items_data(tuple): must contain:
            - item (domish.Element)
            - access_model (unicode)
            - access_list (dict as returned getItemsById, or item_config)
        @param node(LeafNode): node hosting the items
        @param subscription(pubsub.Subscription, None): TODO

        @return (tuple): will contain:
            - notifications_filtered
            - node_owner_jid
            - items_data
        """

        def filterNotifications(result):
            """Check access of subscriber for each item, and keep only allowed ones"""
            notifications, (owner_jid,roster) = result

            #we filter items not allowed for the subscribers
            notifications_filtered = []

            for subscriber, subscriptions, items_data in notifications:
                if subscriber == owner_jid:
                    # as notification is always sent to owner,
                    # we ignore owner if he is here
                    continue
                allowed_items = [] #we keep only item which subscriber can access

                for item_data in items_data:
                    item, access_model = item_data.item, item_data.access_model
                    access_list = item_data.config
                    if access_model == const.VAL_AMODEL_OPEN:
                        allowed_items.append(item)
                    elif access_model == const.VAL_AMODEL_ROSTER:
                        _subscriber = subscriber.userhostJID()
                        if not _subscriber in roster:
                            continue
                        #the subscriber is known, is he in the right group ?
                        authorized_groups = access_list[const.OPT_ROSTER_GROUPS_ALLOWED]
                        if roster[_subscriber].groups.intersection(authorized_groups):
                            allowed_items.append(item)

                    else: #unknown access_model
                        raise NotImplementedError

                if allowed_items:
                    notifications_filtered.append((subscriber, subscriptions, allowed_items))
            return (owner_jid, notifications_filtered)


        if subscription is None:
            d1 = self.backend.getNotifications(node.nodeDbId, items_data)
        else:
            d1 = defer.succeed([(subscription.subscriber, [subscription],
                                items_data)])

        def _got_owner(owner_jid):
            #return a tuple with owner_jid and roster
            def rosterEb(failure):
                log.msg("Error while getting roster of {}: {}".format(unicode(owner_jid), failure.value))
                return (owner_jid, {})

            d = self.backend.privilege.getRoster(owner_jid)
            d.addErrback(rosterEb)
            d.addCallback(lambda roster: (owner_jid,roster))
            return d

        d2 = node.getNodeOwner()
        d2.addCallback(_got_owner)
        d = defer.gatherResults([d1, d2])
        d.addCallback(filterNotifications)
        return d

    def _preDelete(self, data, pep, recipient):
        nodeIdentifier = data['node'].nodeIdentifier
        redirectURI = data.get('redirectURI', None)
        d = self.backend.getSubscribers(nodeIdentifier, pep, recipient)
        d.addCallback(lambda subscribers: self.pubsubService.notifyDelete(
                                                self.serviceJID,
                                                nodeIdentifier,
                                                subscribers,
                                                redirectURI))
        return d


    def _mapErrors(self, failure):
        e = failure.trap(*self._errorMap.keys())

        condition, pubsubCondition, feature = self._errorMap[e]
        msg = failure.value.msg

        if pubsubCondition:
            exc = pubsub.PubSubError(condition, pubsubCondition, feature, msg)
        else:
            exc = StanzaError(condition, text=msg)

        raise exc

    def getInfo(self, requestor, service, nodeIdentifier, pep=None, recipient=None):
        return [] # FIXME: disabled for now, need to manage PEP
        if not requestor.resource:
            # this avoid error when getting a disco request from server during namespace delegation
            return []
        info = {}

        def saveType(result):
            info['type'] = result
            return nodeIdentifier

        def saveMetaData(result):
            info['meta-data'] = result
            return info

        def trapNotFound(failure):
            failure.trap(error.NodeNotFound)
            return info

        d = defer.succeed(nodeIdentifier)
        d.addCallback(self.backend.getNodeType)
        d.addCallback(saveType)
        d.addCallback(self.backend.getNodeMetaData)
        d.addCallback(saveMetaData)
        d.addErrback(trapNotFound)
        d.addErrback(self._mapErrors)
        return d


    def getNodes(self, requestor, service, nodeIdentifier, pep=None):
        return defer.succeed([]) # FIXME: disabled for now, need to manage PEP
        if service.resource:
            return defer.succeed([])
        d = self.backend.getNodes(pep)
        return d.addErrback(self._mapErrors)


    def getConfigurationOptions(self):
        return self.backend.nodeOptions

    def _publish_errb(self, failure, request):
        if failure.type == error.NodeNotFound and self.backend.supportsAutoCreate():
            print "Auto-creating node %s" % (request.nodeIdentifier,)
            d = self.backend.createNode(request.nodeIdentifier,
                                        request.sender,
                                        pep=self._isPep(request),
                                        recipient=request.recipient)
            d.addCallback(lambda ignore,
                                 request: self.backend.publish(request.nodeIdentifier,
                                                               request.items,
                                                               request.sender,
                                                               self._isPep(request),
                                                               request.recipient,
                                                              ),
                          request)
            return d

        return failure

    def _isPep(self, request):
        try:
            return request.delegated
        except AttributeError:
            return False

    def publish(self, request):
        d = self.backend.publish(request.nodeIdentifier,
                                 request.items,
                                 request.sender,
                                 self._isPep(request),
                                 request.recipient)
        d.addErrback(self._publish_errb, request)
        return d.addErrback(self._mapErrors)


    def subscribe(self, request):
        d = self.backend.subscribe(request.nodeIdentifier,
                                   request.subscriber,
                                   request.sender,
                                   self._isPep(request),
                                   request.recipient)
        return d.addErrback(self._mapErrors)


    def unsubscribe(self, request):
        d = self.backend.unsubscribe(request.nodeIdentifier,
                                     request.subscriber,
                                     request.sender,
                                     self._isPep(request),
                                     request.recipient)
        return d.addErrback(self._mapErrors)


    def subscriptions(self, request):
        d = self.backend.getSubscriptions(self._isPep(request),
                                          request.sender)
        return d.addErrback(self._mapErrors)


    def affiliations(self, request):
        d = self.backend.getAffiliations(self._isPep(request),
                                         request.sender)
        return d.addErrback(self._mapErrors)


    def create(self, request):
        d = self.backend.createNode(request.nodeIdentifier,
                                    request.sender, request.options,
                                    self._isPep(request),
                                    request.recipient)
        return d.addErrback(self._mapErrors)


    def default(self, request):
        d = self.backend.getDefaultConfiguration(request.nodeType,
                                                 self._isPep(request),
                                                 request.sender)
        return d.addErrback(self._mapErrors)


    def configureGet(self, request):
        d = self.backend.getNodeConfiguration(request.nodeIdentifier,
                                              self._isPep(request),
                                              request.recipient)
        return d.addErrback(self._mapErrors)


    def configureSet(self, request):
        d = self.backend.setNodeConfiguration(request.nodeIdentifier,
                                              request.options,
                                              request.sender,
                                              self._isPep(request),
                                              request.recipient)
        return d.addErrback(self._mapErrors)


    def items(self, request):
        ext_data = {}
        if const.FLAG_ENABLE_RSM and request.rsm is not None:
            ext_data['rsm'] = request.rsm
        try:
            ext_data['pep'] = request.delegated
        except AttributeError:
            pass
        d = self.backend.getItems(request.nodeIdentifier,
                                  request.recipient,
                                  request.maxItems,
                                  request.itemIdentifiers,
                                  ext_data)
        return d.addErrback(self._mapErrors)

    def retract(self, request):
        d = self.backend.retractItem(request.nodeIdentifier,
                                     request.itemIdentifiers,
                                     request.sender,
                                     request.notify,
                                     self._isPep(request),
                                     request.recipient)
        return d.addErrback(self._mapErrors)


    def purge(self, request):
        d = self.backend.purgeNode(request.nodeIdentifier,
                                   request.sender,
                                   self._isPep(request),
                                   request.recipient)
        return d.addErrback(self._mapErrors)


    def delete(self, request):
        d = self.backend.deleteNode(request.nodeIdentifier,
                                    request.sender,
                                    self._isPep(request),
                                    request.recipient)
        return d.addErrback(self._mapErrors)

components.registerAdapter(PubSubResourceFromBackend,
                           iidavoll.IBackendService,
                           iwokkel.IPubSubResource)