changeset 1616:6bfeb9f0fb84

browser (calls): conferences implementation: - Handle A/V conferences calls creation/joining by entering a conference room JID in the search box. - Group call box has been improved and is used both for group calls (small number of participants) and A/V conferences (larger number of participants). - Fullscreen button for group call is working. - Avatar/user nickname are shown in group call on peer user, as an overlay on video stream. - Use `user` metadata when present to display the right user avatar/name when receiving a stream from SFU (i.e. A/V conference). - Peer user have a new 3 dots menu with a `pin` item to (un)pin it (i.e. display it on full container with on top). - Updated webrtc to handle unidirectional streams correctly and to adapt to A/V conference specification. rel 448
author Goffi <goffi@goffi.org>
date Wed, 07 Aug 2024 00:01:57 +0200
parents 97ea776df74c
children e338426ed4de
files libervia/web/pages/calls/_browser/__init__.py libervia/web/pages/calls/_browser/webrtc.py
diffstat 2 files changed, 177 insertions(+), 22 deletions(-) [+]
line wrap: on
line diff
--- a/libervia/web/pages/calls/_browser/__init__.py	Tue Aug 06 23:56:18 2024 +0200
+++ b/libervia/web/pages/calls/_browser/__init__.py	Wed Aug 07 00:01:57 2024 +0200
@@ -13,6 +13,7 @@
 
 log.warning = log.warn
 profile = window.profile or ""
+own_jid = JID(window.own_jid)
 bridge = Bridge()
 GATHER_TIMEOUT = 10000
 ALLOWED_STATUSES = (
@@ -49,12 +50,12 @@
         )
         # mapping of file sending
         self.files_webrtc: list[dict] = []
-        # WebRTC instances used for group calls.
         self.mode = "search"
         self._status = None
         self._callee: JID|None = None
         self._group_call_room: JID|None = None
         self._group_call_peers: dict = {}
+        self.is_conference = False
         self.contacts_elt = document["contacts"]
         self.search_container_elt = document["search_container"]
         self.call_container_elt = document["call_container"]
@@ -90,6 +91,14 @@
         document["exit_full_screen_btn"].bind(
             "click", lambda __: self.toggle_fullscreen()
         )
+        document["group_full_screen_btn"].bind(
+            "click",
+            lambda __: self.toggle_fullscreen(group=True)
+        )
+        document["group_exit_full_screen_btn"].bind(
+            "click",
+            lambda __: self.toggle_fullscreen(group=True)
+        )
         document["mute_audio_btn"].bind("click", self.toggle_audio_mute)
         document["mute_video_btn"].bind("click", self.toggle_video_mute)
         self.share_desktop_col_elt = document["share_desktop_column"]
@@ -242,6 +251,8 @@
             return
         elif MUJI_PREFIX in action_data.get("from_jid", ""):
             aio.run(self.on_group_call_join(action_data, action_id))
+        elif self.is_conference and action_data["from_jid"] == self._callee:
+            aio.run(self.on_conference_call_join(action_data, action_id))
         else:
             aio.run(self.on_action_new(action_data, action_id))
 
@@ -387,6 +398,7 @@
                 f"Refusing group call join as were are not expecting any from this room.\n"
                 f"{peer_jid.bare=} {self._group_call_room=}"
             )
+            return
         log.info(f"{peer_jid} joined the group call.")
 
         group_video_grid_elt = document["group_video_grid"]
@@ -394,9 +406,12 @@
 
         group_peer_elt = self.group_peer_tpl.get_elt({
             "entity": str(peer_jid),
-            # "identities": cache.identities,
+            "identities": cache.identities,
         })
+        for pin_item in group_peer_elt.select('.action_pin'):
+            pin_item.bind('click', self.toggle_pin)
         group_video_grid_elt <= group_peer_elt
+
         peer_video_stream_elt = group_peer_elt.select_one(".peer_video_stream")
         assert peer_video_stream_elt is not None
         webrtc = WebRTC(
@@ -413,6 +428,42 @@
 
         await bridge.action_launch(action_id, json.dumps({"cancelled": False}))
 
+    async def on_conference_call_join(self, action_data: dict, action_id: str) -> None:
+        log.debug(f"on_conference_call_join {action_data=}")
+        peer_jid = JID(action_data["from_jid"])
+        try:
+            user_jid = JID(action_data["metadata"]["user"])
+        except KeyError:
+            user_jid = peer_jid
+        log.info(f"{user_jid} joined the conference call.")
+
+        group_video_grid_elt = document["group_video_grid"]
+        await cache.fill_identities([str(user_jid)])
+
+        group_peer_elt = self.group_peer_tpl.get_elt({
+            "entity": str(user_jid),
+            "identities": cache.identities,
+        })
+        for pin_item in group_peer_elt.select('.action_pin'):
+            pin_item.bind('click', self.toggle_pin)
+        group_video_grid_elt <= group_peer_elt
+        peer_video_stream_elt = group_peer_elt.select_one(".peer_video_stream")
+        assert peer_video_stream_elt is not None
+        log.debug(f"starting webrtc for {peer_jid} on {peer_video_stream_elt}")
+        webrtc = WebRTC(
+            remote_video_elt=peer_video_stream_elt
+        )
+        sid = webrtc.sid = action_data["session_id"]
+        self._group_call_peers[peer_jid] = {
+            "webrtc": webrtc,
+            "element": group_peer_elt,
+            "sid": sid
+        }
+
+        log.debug(f"Somebody joined conference call. SID: {sid}")
+
+        await bridge.action_launch(action_id, json.dumps({"cancelled": False}))
+
     async def on_action_new(self, action_data: dict, action_id: str) -> None:
         peer_jid = JID(action_data["from_jid"]).bare
         call_type = action_data["sub_type"]
@@ -557,7 +608,9 @@
         if role == "initiator":
             await webrtc.accept_call(session_id, sdp, profile)
         elif role == "responder":
-            await webrtc.answer_call(session_id, sdp, profile)
+            await webrtc.answer_call(
+                session_id, sdp, profile, conference=self.is_conference
+            )
         else:
             dialog.notification.show(
                 f"Invalid role received during setup: {setup_data}", level="error"
@@ -631,8 +684,10 @@
             await cache.fill_identities([entity_jid])
             group_peer_elt = self.group_peer_tpl.get_elt({
                 "entity": str(entity_jid),
-                # "identities": cache.identities,
+                "identities": cache.identities,
             })
+            for pin_item in group_peer_elt.select('.action_pin'):
+                pin_item.bind('click', self.toggle_pin)
             group_video_grid_elt <= group_peer_elt
             peer_video_stream_elt = group_peer_elt.select_one(".peer_video_stream")
             assert peer_video_stream_elt is not None
@@ -669,6 +724,8 @@
         """Call when webRTC connection is reset, we reset buttons statuses"""
         document["full_screen_btn"].classList.remove("is-hidden")
         document["exit_full_screen_btn"].classList.add("is-hidden")
+        document["group_full_screen_btn"].classList.remove("is-hidden")
+        document["group_exit_full_screen_btn"].classList.add("is-hidden")
         for btn_elt in document["mute_audio_btn"], document["mute_video_btn"]:
             btn_elt.classList.remove(INACTIVE_CLASS, MUTED_CLASS, "is-warning")
             btn_elt.classList.add("is-success")
@@ -703,16 +760,55 @@
 
         self._callee = callee_jid
         await cache.fill_identities([callee_jid])
+
+        self.is_conference = False
+        try:
+            disco_identities = cache.identities[callee_jid]["identities"]
+        except KeyError:
+            pass
+        else:
+            for disco_identity in disco_identities:
+                if (
+                    disco_identity.get("category") == "conference"
+                    and disco_identity.get("type") == "audio-video"
+                ):
+                    self.is_conference = True
+                    log.info(f"{callee_jid} is an A/V Conference room.")
+
+
         self.status = "dialing"
         self.set_avatar(callee_jid)
 
-        self.switch_mode("call")
+        self.switch_mode("group_call" if self.is_conference else "call" )
         if remote:
             await self.webrtc.start_remote_control(
                 callee_jid, audio, video
             )
         else:
-            await self.webrtc.make_call(callee_jid, audio, video)
+            if self.is_conference:
+                direction = "sendonly"
+                group_video_grid_elt = document["group_video_grid"]
+                await cache.fill_identities([str(own_jid.bare)])
+                group_peer_elt = self.group_peer_tpl.get_elt({
+                    "entity": str(own_jid.bare),
+                    "identities": cache.identities,
+                })
+                for pin_item in group_peer_elt.select('.action_pin'):
+                    pin_item.bind('click', self.toggle_pin)
+                group_video_grid_elt <= group_peer_elt
+                peer_video_stream_elt = group_peer_elt.select_one(".peer_video_stream")
+                assert peer_video_stream_elt is not None
+                self.webrtc.local_video_elt = peer_video_stream_elt
+            else:
+                direction = "sendrecv"
+                self.webrtc.local_video_elt = document["local_video"]
+
+            await self.webrtc.make_call(
+                callee_jid,
+                audio,
+                video,
+                direction
+            )
 
     async def make_group_call(
         self,
@@ -878,7 +974,7 @@
         for elt_id in to_show:
             document[elt_id].parent.classList.remove("is-hidden")
 
-    def toggle_fullscreen(self, fullscreen: bool | None = None):
+    def toggle_fullscreen(self, fullscreen: bool | None = None, group=False):
         """Toggle fullscreen mode for video elements.
 
         @param fullscreen: if set, determine the fullscreen state; otherwise,
@@ -886,24 +982,59 @@
         """
         if fullscreen is None:
             fullscreen = document.fullscreenElement is NULL
+        log.debug(f"toggle_fullscreen {fullscreen=} {group=}")
+
+        full_screen_cls = "full_screen_btn"
+        exit_full_screen_cls = "exit_full_screen_btn"
+        if group:
+            full_screen_cls = f"group_{full_screen_cls}"
+            exit_full_screen_cls = f"group_{exit_full_screen_cls}"
+            parent_elt = self.group_call_container_elt
+        else:
+            parent_elt = self.call_box_elt
 
         try:
             if fullscreen:
                 if document.fullscreenElement is NULL:
-                    self.call_box_elt.requestFullscreen()
-                    document["full_screen_btn"].classList.add("is-hidden")
-                    document["exit_full_screen_btn"].classList.remove("is-hidden")
+                    parent_elt.requestFullscreen()
+                    document[full_screen_cls].classList.add("is-hidden")
+                    document[exit_full_screen_cls].classList.remove("is-hidden")
             else:
                 if document.fullscreenElement is not NULL:
                     document.exitFullscreen()
-                    document["full_screen_btn"].classList.remove("is-hidden")
-                    document["exit_full_screen_btn"].classList.add("is-hidden")
+                    document[full_screen_cls].classList.remove("is-hidden")
+                    document[exit_full_screen_cls].classList.add("is-hidden")
 
         except Exception as e:
             dialog.notification.show(
                 f"An error occurred while toggling fullscreen: {e}", level="error"
             )
 
+    def toggle_pin(self, event):
+        peer_container = event.target.closest('.peer_video_container')
+        is_pinned = peer_container.dataset.pinned == 'true'
+
+        # Unpin all peers
+        for container in self.group_call_container_elt.select('.peer_video_container'):
+            container.dataset.pinned = 'false'
+            container.classList.remove('is-12')
+            container.classList.add('is-3-widescreen', 'is-4-desktop', 'is-6-tablet')
+
+        if not is_pinned:
+            # Pin the selected peer
+            peer_container.dataset.pinned = 'true'
+            peer_container.classList.remove('is-3-widescreen', 'is-4-desktop', 'is-6-tablet')
+            peer_container.classList.add('is-12')
+
+        # Rearrange the grid
+        grid = document['group_video_grid']
+        pinned = [peer for peer in grid.children if peer.dataset.pinned == 'true']
+        unpinned = [peer for peer in grid.children if peer.dataset.pinned != 'true']
+
+        grid.clear()
+        for peer in pinned + unpinned:
+            grid <= peer
+
     def toggle_audio_mute(self, evt):
         is_muted = self.webrtc.toggle_audio_mute()
         btn_elt = evt.currentTarget
--- a/libervia/web/pages/calls/_browser/webrtc.py	Tue Aug 06 23:56:18 2024 +0200
+++ b/libervia/web/pages/calls/_browser/webrtc.py	Wed Aug 07 00:01:57 2024 +0200
@@ -648,12 +648,19 @@
         window.pc = self._peer_connection
         return peer_connection
 
-    async def _get_user_media(self, audio: bool = True, video: bool = True) -> None:
+    async def _get_user_media(
+        self,
+        audio: bool = True,
+        video: bool = True,
+        direction: str = "sendrecv"
+    ) -> None:
         """
         Gets user media (camera and microphone).
 
         @param audio: True if an audio flux is required.
         @param video: True if a video flux is required.
+        @param direction: The direction of the stream ('sendonly', 'recvonly', 'sendrecv',
+            or 'inactive')
         """
         media_constraints = {"audio": audio, "video": video}
         if self.local_stream is None:
@@ -668,8 +675,12 @@
         if self.local_video_elt is not None:
             self.local_video_elt.srcObject = self.local_stream
 
-        for track in self.local_stream.getTracks():
-            self._peer_connection.addTrack(track)
+        if direction != "recvonly":
+            for track in self.local_stream.getTracks():
+                sender = self._peer_connection.addTransceiver(track, {
+                    'direction': direction,
+                    'streams': [self.local_stream]
+                })
 
     async def _replace_user_video(
         self,
@@ -936,7 +947,7 @@
             self.remote_stream.addTrack(event.track)
 
     def on_negotiation_needed(self, event) -> None:
-        log.debug(f"on_negotiation_needed {event=}")
+        log.debug("on_negotiation_needed")
         # TODO
 
     def _on_data_channel(self, event) -> None:
@@ -947,7 +958,13 @@
         data_channel = event.channel
         self.file_receiver = FileReceiver(self.sid, data_channel, self.extra_data)
 
-    async def answer_call(self, sid: str, offer_sdp: str, profile: str):
+    async def answer_call(
+        self,
+        sid: str,
+        offer_sdp: str,
+        profile: str,
+        conference: bool = False
+    ):
         """We respond to the call"""
         log.debug("answering call")
         if sid != self.sid:
@@ -962,7 +979,9 @@
         if self.file_only:
             self._peer_connection.bind("datachannel", self._on_data_channel)
         else:
-            await self._get_user_media()
+            await self._get_user_media(
+                direction="recvonly" if conference else "sendrecv"
+            )
 
         # Gather local ICE candidates
         local_ice_data = await self._gather_ice_candidates(False)
@@ -998,7 +1017,8 @@
     async def prepare_call(
         self,
         audio: bool = True,
-        video: bool = True
+        video: bool = True,
+        direction: str = "sendrecv"
     ) -> dict:
         """Prepare a call.
 
@@ -1009,17 +1029,21 @@
         @return: Call Data
         """
         await self._create_peer_connection()
-        await self._get_user_media(audio, video)
+        await self._get_user_media(audio, video, direction)
         return await self._get_call_data()
 
     async def make_call(
-        self, callee_jid: jid.JID, audio: bool = True, video: bool = True
+        self,
+        callee_jid: jid.JID,
+        audio: bool = True,
+        video: bool = True,
+        direction: str = "sendrecv"
     ) -> None:
         """
         @param audio: True if an audio flux is required
         @param video: True if a video flux is required
         """
-        call_data = await self.prepare_call(audio, video)
+        call_data = await self.prepare_call(audio, video, direction)
         log.info(f"calling {callee_jid!r}")
         self.sid = await bridge.call_start(str(callee_jid), json.dumps(call_data))
         log.debug(f"Call SID: {self.sid}")