# HG changeset patch # User Goffi # Date 1722981717 -7200 # Node ID 6bfeb9f0fb84db8825b8e506c39ceb65171e5a21 # Parent 97ea776df74c0319ac27027f77065ab79d73564d browser (calls): conferences implementation: - Handle A/V conferences calls creation/joining by entering a conference room JID in the search box. - Group call box has been improved and is used both for group calls (small number of participants) and A/V conferences (larger number of participants). - Fullscreen button for group call is working. - Avatar/user nickname are shown in group call on peer user, as an overlay on video stream. - Use `user` metadata when present to display the right user avatar/name when receiving a stream from SFU (i.e. A/V conference). - Peer user have a new 3 dots menu with a `pin` item to (un)pin it (i.e. display it on full container with on top). - Updated webrtc to handle unidirectional streams correctly and to adapt to A/V conference specification. rel 448 diff -r 97ea776df74c -r 6bfeb9f0fb84 libervia/web/pages/calls/_browser/__init__.py --- a/libervia/web/pages/calls/_browser/__init__.py Tue Aug 06 23:56:18 2024 +0200 +++ b/libervia/web/pages/calls/_browser/__init__.py Wed Aug 07 00:01:57 2024 +0200 @@ -13,6 +13,7 @@ log.warning = log.warn profile = window.profile or "" +own_jid = JID(window.own_jid) bridge = Bridge() GATHER_TIMEOUT = 10000 ALLOWED_STATUSES = ( @@ -49,12 +50,12 @@ ) # mapping of file sending self.files_webrtc: list[dict] = [] - # WebRTC instances used for group calls. self.mode = "search" self._status = None self._callee: JID|None = None self._group_call_room: JID|None = None self._group_call_peers: dict = {} + self.is_conference = False self.contacts_elt = document["contacts"] self.search_container_elt = document["search_container"] self.call_container_elt = document["call_container"] @@ -90,6 +91,14 @@ document["exit_full_screen_btn"].bind( "click", lambda __: self.toggle_fullscreen() ) + document["group_full_screen_btn"].bind( + "click", + lambda __: self.toggle_fullscreen(group=True) + ) + document["group_exit_full_screen_btn"].bind( + "click", + lambda __: self.toggle_fullscreen(group=True) + ) document["mute_audio_btn"].bind("click", self.toggle_audio_mute) document["mute_video_btn"].bind("click", self.toggle_video_mute) self.share_desktop_col_elt = document["share_desktop_column"] @@ -242,6 +251,8 @@ return elif MUJI_PREFIX in action_data.get("from_jid", ""): aio.run(self.on_group_call_join(action_data, action_id)) + elif self.is_conference and action_data["from_jid"] == self._callee: + aio.run(self.on_conference_call_join(action_data, action_id)) else: aio.run(self.on_action_new(action_data, action_id)) @@ -387,6 +398,7 @@ f"Refusing group call join as were are not expecting any from this room.\n" f"{peer_jid.bare=} {self._group_call_room=}" ) + return log.info(f"{peer_jid} joined the group call.") group_video_grid_elt = document["group_video_grid"] @@ -394,9 +406,12 @@ group_peer_elt = self.group_peer_tpl.get_elt({ "entity": str(peer_jid), - # "identities": cache.identities, + "identities": cache.identities, }) + for pin_item in group_peer_elt.select('.action_pin'): + pin_item.bind('click', self.toggle_pin) group_video_grid_elt <= group_peer_elt + peer_video_stream_elt = group_peer_elt.select_one(".peer_video_stream") assert peer_video_stream_elt is not None webrtc = WebRTC( @@ -413,6 +428,42 @@ await bridge.action_launch(action_id, json.dumps({"cancelled": False})) + async def on_conference_call_join(self, action_data: dict, action_id: str) -> None: + log.debug(f"on_conference_call_join {action_data=}") + peer_jid = JID(action_data["from_jid"]) + try: + user_jid = JID(action_data["metadata"]["user"]) + except KeyError: + user_jid = peer_jid + log.info(f"{user_jid} joined the conference call.") + + group_video_grid_elt = document["group_video_grid"] + await cache.fill_identities([str(user_jid)]) + + group_peer_elt = self.group_peer_tpl.get_elt({ + "entity": str(user_jid), + "identities": cache.identities, + }) + for pin_item in group_peer_elt.select('.action_pin'): + pin_item.bind('click', self.toggle_pin) + group_video_grid_elt <= group_peer_elt + peer_video_stream_elt = group_peer_elt.select_one(".peer_video_stream") + assert peer_video_stream_elt is not None + log.debug(f"starting webrtc for {peer_jid} on {peer_video_stream_elt}") + webrtc = WebRTC( + remote_video_elt=peer_video_stream_elt + ) + sid = webrtc.sid = action_data["session_id"] + self._group_call_peers[peer_jid] = { + "webrtc": webrtc, + "element": group_peer_elt, + "sid": sid + } + + log.debug(f"Somebody joined conference call. SID: {sid}") + + await bridge.action_launch(action_id, json.dumps({"cancelled": False})) + async def on_action_new(self, action_data: dict, action_id: str) -> None: peer_jid = JID(action_data["from_jid"]).bare call_type = action_data["sub_type"] @@ -557,7 +608,9 @@ if role == "initiator": await webrtc.accept_call(session_id, sdp, profile) elif role == "responder": - await webrtc.answer_call(session_id, sdp, profile) + await webrtc.answer_call( + session_id, sdp, profile, conference=self.is_conference + ) else: dialog.notification.show( f"Invalid role received during setup: {setup_data}", level="error" @@ -631,8 +684,10 @@ await cache.fill_identities([entity_jid]) group_peer_elt = self.group_peer_tpl.get_elt({ "entity": str(entity_jid), - # "identities": cache.identities, + "identities": cache.identities, }) + for pin_item in group_peer_elt.select('.action_pin'): + pin_item.bind('click', self.toggle_pin) group_video_grid_elt <= group_peer_elt peer_video_stream_elt = group_peer_elt.select_one(".peer_video_stream") assert peer_video_stream_elt is not None @@ -669,6 +724,8 @@ """Call when webRTC connection is reset, we reset buttons statuses""" document["full_screen_btn"].classList.remove("is-hidden") document["exit_full_screen_btn"].classList.add("is-hidden") + document["group_full_screen_btn"].classList.remove("is-hidden") + document["group_exit_full_screen_btn"].classList.add("is-hidden") for btn_elt in document["mute_audio_btn"], document["mute_video_btn"]: btn_elt.classList.remove(INACTIVE_CLASS, MUTED_CLASS, "is-warning") btn_elt.classList.add("is-success") @@ -703,16 +760,55 @@ self._callee = callee_jid await cache.fill_identities([callee_jid]) + + self.is_conference = False + try: + disco_identities = cache.identities[callee_jid]["identities"] + except KeyError: + pass + else: + for disco_identity in disco_identities: + if ( + disco_identity.get("category") == "conference" + and disco_identity.get("type") == "audio-video" + ): + self.is_conference = True + log.info(f"{callee_jid} is an A/V Conference room.") + + self.status = "dialing" self.set_avatar(callee_jid) - self.switch_mode("call") + self.switch_mode("group_call" if self.is_conference else "call" ) if remote: await self.webrtc.start_remote_control( callee_jid, audio, video ) else: - await self.webrtc.make_call(callee_jid, audio, video) + if self.is_conference: + direction = "sendonly" + group_video_grid_elt = document["group_video_grid"] + await cache.fill_identities([str(own_jid.bare)]) + group_peer_elt = self.group_peer_tpl.get_elt({ + "entity": str(own_jid.bare), + "identities": cache.identities, + }) + for pin_item in group_peer_elt.select('.action_pin'): + pin_item.bind('click', self.toggle_pin) + group_video_grid_elt <= group_peer_elt + peer_video_stream_elt = group_peer_elt.select_one(".peer_video_stream") + assert peer_video_stream_elt is not None + self.webrtc.local_video_elt = peer_video_stream_elt + else: + direction = "sendrecv" + self.webrtc.local_video_elt = document["local_video"] + + await self.webrtc.make_call( + callee_jid, + audio, + video, + direction + ) async def make_group_call( self, @@ -878,7 +974,7 @@ for elt_id in to_show: document[elt_id].parent.classList.remove("is-hidden") - def toggle_fullscreen(self, fullscreen: bool | None = None): + def toggle_fullscreen(self, fullscreen: bool | None = None, group=False): """Toggle fullscreen mode for video elements. @param fullscreen: if set, determine the fullscreen state; otherwise, @@ -886,24 +982,59 @@ """ if fullscreen is None: fullscreen = document.fullscreenElement is NULL + log.debug(f"toggle_fullscreen {fullscreen=} {group=}") + + full_screen_cls = "full_screen_btn" + exit_full_screen_cls = "exit_full_screen_btn" + if group: + full_screen_cls = f"group_{full_screen_cls}" + exit_full_screen_cls = f"group_{exit_full_screen_cls}" + parent_elt = self.group_call_container_elt + else: + parent_elt = self.call_box_elt try: if fullscreen: if document.fullscreenElement is NULL: - self.call_box_elt.requestFullscreen() - document["full_screen_btn"].classList.add("is-hidden") - document["exit_full_screen_btn"].classList.remove("is-hidden") + parent_elt.requestFullscreen() + document[full_screen_cls].classList.add("is-hidden") + document[exit_full_screen_cls].classList.remove("is-hidden") else: if document.fullscreenElement is not NULL: document.exitFullscreen() - document["full_screen_btn"].classList.remove("is-hidden") - document["exit_full_screen_btn"].classList.add("is-hidden") + document[full_screen_cls].classList.remove("is-hidden") + document[exit_full_screen_cls].classList.add("is-hidden") except Exception as e: dialog.notification.show( f"An error occurred while toggling fullscreen: {e}", level="error" ) + def toggle_pin(self, event): + peer_container = event.target.closest('.peer_video_container') + is_pinned = peer_container.dataset.pinned == 'true' + + # Unpin all peers + for container in self.group_call_container_elt.select('.peer_video_container'): + container.dataset.pinned = 'false' + container.classList.remove('is-12') + container.classList.add('is-3-widescreen', 'is-4-desktop', 'is-6-tablet') + + if not is_pinned: + # Pin the selected peer + peer_container.dataset.pinned = 'true' + peer_container.classList.remove('is-3-widescreen', 'is-4-desktop', 'is-6-tablet') + peer_container.classList.add('is-12') + + # Rearrange the grid + grid = document['group_video_grid'] + pinned = [peer for peer in grid.children if peer.dataset.pinned == 'true'] + unpinned = [peer for peer in grid.children if peer.dataset.pinned != 'true'] + + grid.clear() + for peer in pinned + unpinned: + grid <= peer + def toggle_audio_mute(self, evt): is_muted = self.webrtc.toggle_audio_mute() btn_elt = evt.currentTarget diff -r 97ea776df74c -r 6bfeb9f0fb84 libervia/web/pages/calls/_browser/webrtc.py --- a/libervia/web/pages/calls/_browser/webrtc.py Tue Aug 06 23:56:18 2024 +0200 +++ b/libervia/web/pages/calls/_browser/webrtc.py Wed Aug 07 00:01:57 2024 +0200 @@ -648,12 +648,19 @@ window.pc = self._peer_connection return peer_connection - async def _get_user_media(self, audio: bool = True, video: bool = True) -> None: + async def _get_user_media( + self, + audio: bool = True, + video: bool = True, + direction: str = "sendrecv" + ) -> None: """ Gets user media (camera and microphone). @param audio: True if an audio flux is required. @param video: True if a video flux is required. + @param direction: The direction of the stream ('sendonly', 'recvonly', 'sendrecv', + or 'inactive') """ media_constraints = {"audio": audio, "video": video} if self.local_stream is None: @@ -668,8 +675,12 @@ if self.local_video_elt is not None: self.local_video_elt.srcObject = self.local_stream - for track in self.local_stream.getTracks(): - self._peer_connection.addTrack(track) + if direction != "recvonly": + for track in self.local_stream.getTracks(): + sender = self._peer_connection.addTransceiver(track, { + 'direction': direction, + 'streams': [self.local_stream] + }) async def _replace_user_video( self, @@ -936,7 +947,7 @@ self.remote_stream.addTrack(event.track) def on_negotiation_needed(self, event) -> None: - log.debug(f"on_negotiation_needed {event=}") + log.debug("on_negotiation_needed") # TODO def _on_data_channel(self, event) -> None: @@ -947,7 +958,13 @@ data_channel = event.channel self.file_receiver = FileReceiver(self.sid, data_channel, self.extra_data) - async def answer_call(self, sid: str, offer_sdp: str, profile: str): + async def answer_call( + self, + sid: str, + offer_sdp: str, + profile: str, + conference: bool = False + ): """We respond to the call""" log.debug("answering call") if sid != self.sid: @@ -962,7 +979,9 @@ if self.file_only: self._peer_connection.bind("datachannel", self._on_data_channel) else: - await self._get_user_media() + await self._get_user_media( + direction="recvonly" if conference else "sendrecv" + ) # Gather local ICE candidates local_ice_data = await self._gather_ice_candidates(False) @@ -998,7 +1017,8 @@ async def prepare_call( self, audio: bool = True, - video: bool = True + video: bool = True, + direction: str = "sendrecv" ) -> dict: """Prepare a call. @@ -1009,17 +1029,21 @@ @return: Call Data """ await self._create_peer_connection() - await self._get_user_media(audio, video) + await self._get_user_media(audio, video, direction) return await self._get_call_data() async def make_call( - self, callee_jid: jid.JID, audio: bool = True, video: bool = True + self, + callee_jid: jid.JID, + audio: bool = True, + video: bool = True, + direction: str = "sendrecv" ) -> None: """ @param audio: True if an audio flux is required @param video: True if a video flux is required """ - call_data = await self.prepare_call(audio, video) + call_data = await self.prepare_call(audio, video, direction) log.info(f"calling {callee_jid!r}") self.sid = await bridge.call_start(str(callee_jid), json.dumps(call_data)) log.debug(f"Call SID: {self.sid}")