Mercurial > libervia-web
changeset 1549:e47c24204449
browser (calls): update call to handle search, control buttons, and better UI/UX:
- adapt to backend changes
- UI and WebRTC parts are not separated
- users can now be searched
- add mute/fullscreen buttons
- ring
- cancellable dialog when a call is received
- status of the call
- animations
- various UI/UX improvments
rel 423
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Wed, 09 Aug 2023 00:22:18 +0200 |
| parents | 66aa6e140ebb |
| children | 4b0464b2a12b |
| files | libervia/web/pages/_browser/dialog.py libervia/web/pages/_browser/jid_search.py libervia/web/pages/calls/_browser/__init__.py libervia/web/pages/calls/_browser/webrtc.py libervia/web/server/restricted_bridge.py libervia/web/server/server.py |
| diffstat | 6 files changed, 945 insertions(+), 414 deletions(-) [+] |
line wrap: on
line diff
--- a/libervia/web/pages/_browser/dialog.py Wed Aug 09 00:22:16 2023 +0200 +++ b/libervia/web/pages/_browser/dialog.py Wed Aug 09 00:22:18 2023 +0200 @@ -6,6 +6,10 @@ log.warning = log.warn +class CancelError(Exception): + """Dialog is cancelled""" + + class Confirm: def __init__(self, message, ok_label="", cancel_label="", ok_color="success"): @@ -15,26 +19,63 @@ assert ok_color in ("success", "danger") self.ok_color = ok_color self.cancel_label = cancel_label + self.reset() - def cancel_cb(self, evt, notif_elt): + def reset(self): + """Reset values of callbacks and notif element""" + self._ok_cb = None + self._cancel_cb = None + self._reject_cb = None + self._notif_elt = None + + def _default_cancel_cb(self, evt, notif_elt): notif_elt.remove() - def show(self, ok_cb, cancel_cb=None): + def cancel(self): + """Cancel the dialog, without calling any callback + + will raise a CancelError + """ + if self._notif_elt is None: + log.warning("calling cancel on an unshown dialog") + else: + self._notif_elt.remove() + if self._reject_cb is not None: + self._reject_cb(CancelError) + else: + log.warning("no reject callback set") + self.reset() + + def on_ok_click(self, evt): + assert self._ok_cb is not None + self._ok_cb(evt, self._notif_elt) + self.reset() + + def on_cancel_click(self, evt) -> None: + assert self._cancel_cb is not None + self._cancel_cb(evt, self._notif_elt) + self.reset() + + def show(self, ok_cb, cancel_cb=None, reject_cb=None): if cancel_cb is None: - cancel_cb = self.cancel_cb + cancel_cb = self._default_cancel_cb + self._ok_cb = ok_cb + self._cancel_cb = cancel_cb + self._reject_cb = reject_cb notif_elt = self._tpl.get_elt({ "message": self.message, "ok_label": self.ok_label, "ok_color": self.ok_color, "cancel_label": self.cancel_label, }) + self._notif_elt = notif_elt document['notifs_area'] <= notif_elt timer.set_timeout(lambda: notif_elt.classList.add('state_appended'), 0) - for cancel_elt in notif_elt.select(".click_to_cancel"): - cancel_elt.bind("click", lambda evt: cancel_cb(evt, notif_elt)) - for cancel_elt in notif_elt.select(".click_to_ok"): - cancel_elt.bind("click", lambda evt: ok_cb(evt, notif_elt)) + for ok_elt in notif_elt.select(".click_to_ok"): + ok_elt.bind("click", self.on_ok_click) + for ok_elt in notif_elt.select(".click_to_cancel"): + ok_elt.bind("click", self.on_cancel_click) def _ashow_cb(self, evt, notif_elt, resolve_cb, confirmed): evt.stopPropagation() @@ -46,7 +87,8 @@ lambda resolve_cb, reject_cb: self.show( lambda evt, notif_elt: self._ashow_cb(evt, notif_elt, resolve_cb, True), - lambda evt, notif_elt: self._ashow_cb(evt, notif_elt, resolve_cb, False) + lambda evt, notif_elt: self._ashow_cb(evt, notif_elt, resolve_cb, False), + reject_cb ) )
--- a/libervia/web/pages/_browser/jid_search.py Wed Aug 09 00:22:16 2023 +0200 +++ b/libervia/web/pages/_browser/jid_search.py Wed Aug 09 00:22:18 2023 +0200 @@ -53,7 +53,9 @@ self.empty_cb = empty_cb or self.on_empty_search current_search = search_elt.value.strip() or None - if current_search is not None: + if not current_search: + self.empty_cb() + else: aio.run(self.perform_search(current_search)) def default_get_url(self, item):
--- a/libervia/web/pages/calls/_browser/__init__.py Wed Aug 09 00:22:16 2023 +0200 +++ b/libervia/web/pages/calls/_browser/__init__.py Wed Aug 09 00:22:18 2023 +0200 @@ -1,279 +1,150 @@ import json -import re from bridge import AsyncBridge as Bridge -from browser import aio, console as log, document, timer, window -import errors +from browser import aio, console as log, document, window +from cache import cache +import dialog +from jid import JID +from jid_search import JidSearch import loading +from template import Template +from webrtc import WebRTC log.warning = log.warn profile = window.profile or "" bridge = Bridge() GATHER_TIMEOUT = 10000 +ALLOWED_STATUSES = (None, "dialing") +AUDIO = 'audio' +VIDEO = 'video' +ALLOWED_CALL_MODES = {AUDIO, VIDEO} -class WebRTCCall: +class CallUI: def __init__(self): - self.reset_instance() - - def reset_instance(self): - """Inits or resets the instance variables to their default state.""" - self._peer_connection = None - self._media_types = None - self.sid = None - self.local_candidates = None - self.remote_stream = None - self.candidates_buffer = { - "audio": {"candidates": []}, - "video": {"candidates": []}, - } - self.media_candidates = {} - self.candidates_gathered = aio.Future() - - @property - def media_types(self): - if self._media_types is None: - raise Exception("self._media_types should not be None!") - return self._media_types - - def get_sdp_mline_index(self, media_type): - """Gets the sdpMLineIndex for a given media type. - - @param media_type: The type of the media. - """ - for index, m_type in self.media_types.items(): - if m_type == media_type: - return index - raise ValueError(f"Media type '{media_type}' not found") - - def extract_pwd_ufrag(self, sdp): - """Retrieves ICE password and user fragment for SDP offer. - - @param sdp: The Session Description Protocol offer string. - """ - ufrag_line = re.search(r"ice-ufrag:(\S+)", sdp) - pwd_line = re.search(r"ice-pwd:(\S+)", sdp) - - if ufrag_line and pwd_line: - return ufrag_line.group(1), pwd_line.group(1) - else: - log.error(f"SDP with missing ice-ufrag or ice-pwd:\n{sdp}") - raise ValueError("Can't extract ice-ufrag and ice-pwd from SDP") - - def extract_fingerprint_data(self, sdp): - """Retrieves fingerprint data from an SDP offer. + self.webrtc = WebRTC() + self.mode = "search" + self._status = None + self._callee = None + self.contacts_elt = document["contacts"] + self.search_container_elt = document["search_container"] + self.call_container_elt = document["call_container"] + self.call_box_elt = document["call_box"] + self.call_avatar_wrapper_elt = document["call_avatar_wrapper"] + self.call_status_wrapper_elt = document["call_status_wrapper"] + self.call_avatar_tpl = Template("call/call_avatar.html") + self.call_status_tpl = Template("call/call_status.html") + self.audio_player_elt = document["audio_player"] + bridge.register_signal("action_new", self._on_action_new) + bridge.register_signal("call_setup", self._on_call_setup) + bridge.register_signal("call_ended", self._on_call_ended) - @param sdp: The Session Description Protocol offer string. - @return: A dictionary containing the fingerprint data. - """ - fingerprint_line = re.search(r"a=fingerprint:(\S+)\s+(\S+)", sdp) - if fingerprint_line: - algorithm, fingerprint = fingerprint_line.groups() - fingerprint_data = { - "hash": algorithm, - "fingerprint": fingerprint - } - - setup_line = re.search(r"a=setup:(\S+)", sdp) - if setup_line: - setup = setup_line.group(1) - fingerprint_data["setup"] = setup - - return fingerprint_data - else: - raise ValueError("fingerprint should not be missing") - - def parse_ice_candidate(self, candidate_string): - """Parses the ice candidate string. + # call/hang up buttons + self._call_mode = VIDEO + self.call_button_tpl = Template("call/call_button.html") + self._update_call_button() + document['toggle_call_mode_btn'].bind('click', self.switch_call_mode) + document["hangup_btn"].bind( + "click", + lambda __: aio.run(self.hang_up()) + ) - @param candidate_string: The ice candidate string to be parsed. - """ - pattern = re.compile( - r"candidate:(?P<foundation>\S+) (?P<component_id>\d+) (?P<transport>\S+) " - r"(?P<priority>\d+) (?P<address>\S+) (?P<port>\d+) typ " - r"(?P<type>\S+)(?: raddr (?P<rel_addr>\S+) rport " - r"(?P<rel_port>\d+))?(?: generation (?P<generation>\d+))?" + # other buttons + document["full_screen_btn"].bind( + "click", + lambda __: self.toggle_fullscreen() + ) + document["exit_full_screen_btn"].bind( + "click", + lambda __: self.toggle_fullscreen() ) - match = pattern.match(candidate_string) - if match: - candidate_dict = match.groupdict() - - # Apply the correct types to the dictionary values - candidate_dict["component_id"] = int(candidate_dict["component_id"]) - candidate_dict["priority"] = int(candidate_dict["priority"]) - candidate_dict["port"] = int(candidate_dict["port"]) + document["mute_audio_btn"].bind( + "click", + self.toggle_audio_mute + ) + document["mute_video_btn"].bind( + "click", + self.toggle_video_mute + ) - if candidate_dict["rel_port"]: - candidate_dict["rel_port"] = int(candidate_dict["rel_port"]) - - if candidate_dict["generation"]: - candidate_dict["generation"] = candidate_dict["generation"] - - # Remove None values - return {k: v for k, v in candidate_dict.items() if v is not None} - else: - log.warning(f"can't parse candidate: {candidate_string!r}") - return None - - def build_ice_candidate(self, parsed_candidate): - """Builds ICE candidate - - @param parsed_candidate: Dictionary containing parsed ICE candidate - """ - base_format = ( - "candidate:{foundation} {component_id} {transport} {priority} " - "{address} {port} typ {type}" + # search + self.jid_search = JidSearch( + document["search"], + document["contacts"], + click_cb = self._on_entity_click ) - if ((parsed_candidate.get('rel_addr') - and parsed_candidate.get('rel_port'))): - base_format += " raddr {rel_addr} rport {rel_port}" + # incoming call dialog + self.incoming_call_dialog_elt = None - if parsed_candidate.get('generation'): - base_format += " generation {generation}" - - return base_format.format(**parsed_candidate) - - def on_ice_candidate(self, event): - """Handles ICE candidate event + @property + def sid(self) -> str|None: + return self.webrtc.sid - @param event: Event containing the ICE candidate - """ - log.debug(f"on ice candidate {event.candidate=}") - if event.candidate and event.candidate.candidate: - window.last_event = event - parsed_candidate = self.parse_ice_candidate(event.candidate.candidate) - if parsed_candidate is None: - return - try: - media_type = self.media_types[event.candidate.sdpMLineIndex] - except (TypeError, IndexError): - log.error( - f"Can't find media type.\n{event.candidate=}\n{self._media_types=}" - ) - return - self.media_candidates.setdefault(media_type, []).append(parsed_candidate) - log.debug(f"ICE candidate [{media_type}]: {event.candidate.candidate}") - else: - log.debug("All ICE candidates gathered") + @sid.setter + def sid(self, new_sid) -> None: + self.webrtc.sid = new_sid + + @property + def status(self): + return self._status - def _set_media_types(self, offer): - """Sets media types from offer SDP - - @param offer: RTC session description containing the offer - """ - sdp_lines = offer.sdp.splitlines() - media_types = {} - mline_index = 0 - - for line in sdp_lines: - if line.startswith("m="): - media_types[mline_index] = line[2:line.find(" ")] - mline_index += 1 - - self._media_types = media_types - - def on_ice_gathering_state_change(self, event): - """Handles ICE gathering state change - - @param event: Event containing the ICE gathering state change - """ - connection = event.target - log.debug(f"on_ice_gathering_state_change {connection.iceGatheringState=}") - if connection.iceGatheringState == "complete": - log.info("ICE candidates gathering done") - self.candidates_gathered.set_result(None) - - async def _create_peer_connection( - self, - ): - """Creates peer connection""" - if self._peer_connection is not None: - raise Exception("create_peer_connection can't be called twice!") - - external_disco = json.loads(await bridge.external_disco_get("")) - ice_servers = [] + @status.setter + def status(self, new_status): + if new_status != self._status: + if new_status not in ALLOWED_STATUSES: + raise Exception( + f"INTERNAL ERROR: this status is not allowed: {new_status!r}" + ) + tpl_data = { + "entity": self._callee, + "status": new_status + } + if self._callee is not None: + try: + tpl_data["name"] = cache.identities[self._callee]["nicknames"][0] + except (KeyError, IndexError): + tpl_data["name"] = str(self._callee) + status_elt = self.call_status_tpl.get_elt(tpl_data) + self.call_status_wrapper_elt.clear() + self.call_status_wrapper_elt <= status_elt - for server in external_disco: - ice_server = {} - if server["type"] == "stun": - ice_server["urls"] = f"stun:{server['host']}:{server['port']}" - elif server["type"] == "turn": - ice_server["urls"] = ( - f"turn:{server['host']}:{server['port']}?transport={server['transport']}" - ) - ice_server["username"] = server["username"] - ice_server["credential"] = server["password"] - ice_servers.append(ice_server) - rtc_configuration = {"iceServers": ice_servers} - - peer_connection = window.RTCPeerConnection.new(rtc_configuration) - peer_connection.addEventListener("track", self.on_track) - peer_connection.addEventListener("negotiationneeded", self.on_negotiation_needed) - peer_connection.addEventListener("icecandidate", self.on_ice_candidate) - peer_connection.addEventListener("icegatheringstatechange", self.on_ice_gathering_state_change) + self._status = new_status - self._peer_connection = peer_connection - window.pc = self._peer_connection - - async def _get_user_media( - self, - audio: bool = True, - video: bool = True - ): - """Gets user media - - @param audio: True if an audio flux is required - @param video: True if a video flux is required - """ - media_constraints = {'audio': audio, 'video': video} - local_stream = await window.navigator.mediaDevices.getUserMedia(media_constraints) - document["local_video"].srcObject = local_stream - - for track in local_stream.getTracks(): - self._peer_connection.addTrack(track) + @property + def call_mode(self): + return self._call_mode - async def _gather_ice_candidates(self, is_initiator: bool, remote_candidates=None): - """Get ICE candidates and wait to have them all before returning them - - @param is_initiator: Boolean indicating if the user is the initiator of the connection - @param remote_candidates: Remote ICE candidates, if any - """ - if self._peer_connection is None: - raise Exception("The peer connection must be created before gathering ICE candidates!") - - self.media_candidates.clear() - gather_timeout = timer.set_timeout( - lambda: self.candidates_gathered.set_exception( - errors.TimeoutError("ICE gathering time out") - ), - GATHER_TIMEOUT - ) + @call_mode.setter + def call_mode(self, mode): + if mode in ALLOWED_CALL_MODES: + if self._call_mode == mode: + return + self._call_mode = mode + self._update_call_button() + with_video = mode == VIDEO + for elt in self.call_box_elt.select(".is-video-only"): + if with_video: + elt.classList.remove("is-hidden") + else: + elt.classList.add("is-hidden") + else: + raise ValueError("Invalid call mode") - if is_initiator: - offer = await self._peer_connection.createOffer() - self._set_media_types(offer) - await self._peer_connection.setLocalDescription(offer) - else: - answer = await self._peer_connection.createAnswer() - self._set_media_types(answer) - await self._peer_connection.setLocalDescription(answer) + def switch_call_mode(self, ev): + self.call_mode = AUDIO if self.call_mode == VIDEO else VIDEO - if not is_initiator: - log.debug(self._peer_connection.localDescription.sdp) - await self.candidates_gathered - log.debug(self._peer_connection.localDescription.sdp) - timer.clear_timeout(gather_timeout) - ufrag, pwd = self.extract_pwd_ufrag(self._peer_connection.localDescription.sdp) - return { - "ufrag": ufrag, - "pwd": pwd, - "candidates": self.media_candidates, - } + def _update_call_button(self): + new_button = self.call_button_tpl.get_elt({"call_mode": self.call_mode}) + new_button.bind( + "click", + lambda __: aio.run(self.make_call(video=not self.call_mode == AUDIO)) + ) + document['call_btn'].replaceWith(new_button) - def on_action_new( + def _on_action_new( self, action_data_s: str, action_id: str, security_limit: int, profile: str ) -> None: """Called when a call is received @@ -286,6 +157,9 @@ action_data = json.loads(action_data_s) if action_data.get("type") != "call": return + aio.run(self.on_action_new(action_data, action_id)) + + async def on_action_new(self, action_data: dict, action_id: str) -> None: peer_jid = action_data["from_jid"] log.info( f"{peer_jid} wants to start a call ({action_data['sub_type']})" @@ -296,21 +170,50 @@ f"{peer_jid}" ) return - self.sid = action_data["session_id"] - log.debug(f"Call SID: {self.sid}") + sid = self.sid = action_data["session_id"] + await cache.fill_identities([peer_jid]) + identity = cache.identities[peer_jid] + peer_name = identity['nicknames'][0] - # Answer the call - offer_sdp = action_data["sdp"] - aio.run(self.answer_call(offer_sdp, action_id)) + # we start the ring + self.audio_player_elt.play() - def _on_call_accepted(self, session_id: str, sdp: str, profile: str) -> None: - """Called when we have received answer SDP from responder + # and ask user if we take the call + try: + self.incoming_call_dialog_elt = dialog.Confirm( + f"{peer_name} is calling you.", + ok_label="Answer", + cancel_label="Reject" + ) + accepted = await self.incoming_call_dialog_elt.ashow() + except dialog.CancelError: + log.info("Call has been cancelled") + self.incoming_call_dialog_elt = None + self.sid = None + dialog.notification.show( + f"{peer_name} has cancelled the call", + level="info" + ) + return - @param session_id: Session identifier - @param sdp: Session Description Protocol data - @param profile: Profile associated with the action - """ - aio.run(self.on_call_accepted(session_id, sdp, profile)) + self.incoming_call_dialog_elt = None + + # we stop the ring + self.audio_player_elt.pause() + self.audio_player_elt.currentTime = 0 + + if accepted: + log.debug(f"Call SID: {sid}") + + # Answer the call + self.switch_mode("call") + else: + log.info(f"your are declining the call from {peer_jid}") + self.sid = None + await bridge.action_launch( + action_id, + json.dumps({"cancelled": not accepted}) + ) def _on_call_ended(self, session_id: str, data_s: str, profile: str) -> None: """Call has been terminated @@ -327,13 +230,24 @@ f"ignoring call_ended not linked to our call ({self.sid}): {session_id}" ) return - aio.run(self.end_call()) + aio.run(self.end_call(json.loads(data_s))) + + def _on_call_setup(self, session_id: str, setup_data_s: str, profile: str) -> None: + """Called when we have received answer SDP from responder - async def on_call_accepted(self, session_id: str, sdp: str, profile: str) -> None: + @param session_id: Session identifier + @param sdp: Session Description Protocol data + @param profile: Profile associated with the action + """ + aio.run(self.on_call_setup(session_id, json.loads(setup_data_s), profile)) + + async def on_call_setup(self, session_id: str, setup_data: dict, profile: str) -> None: """Call has been accepted, connection can be established @param session_id: Session identifier - @param sdp: Session Description Protocol data + @param setup_data: Data with following keys: + role: initiator or responser + sdp: Session Description Protocol data @param profile: Profile associated """ if self.sid != session_id: @@ -341,103 +255,25 @@ f"Call ignored due to different session ID ({self.sid=} {session_id=})" ) return - await self._peer_connection.setRemoteDescription({ - "type": "answer", - "sdp": sdp - }) - await self.on_ice_candidates_new(self.candidates_buffer) - self.candidates_buffer.clear() - - def _on_ice_candidates_new(self, sid: str, candidates_s: str, profile: str) -> None: - """Called when new ICE candidates are received - - @param sid: Session identifier - @param candidates_s: ICE candidates serialized - @param profile: Profile associated with the action - """ - if sid != self.sid: - log.debug( - f"ignoring peer ice candidates for {sid=} ({self.sid=})." + try: + role = setup_data["role"] + sdp = setup_data["sdp"] + except KeyError: + dialog.notification.show( + f"Invalid setup data received: {setup_data}", + level="error" ) return - candidates = json.loads(candidates_s) - aio.run(self.on_ice_candidates_new(candidates)) - - async def on_ice_candidates_new(self, candidates: dict) -> None: - """Called when new ICE canidates are received from peer - - @param candidates: Dictionary containing new ICE candidates - """ - log.debug(f"new peer candidates received: {candidates}") - if ( - self._peer_connection is None - or self._peer_connection.remoteDescription is None - ): - for media_type in ("audio", "video"): - media_candidates = candidates.get(media_type) - if media_candidates: - buffer = self.candidates_buffer[media_type] - buffer["candidates"].extend(media_candidates["candidates"]) + if role == "initiator": + await self.webrtc.accept_call(session_id, sdp, profile) + elif role == "responder": + await self.webrtc.answer_call(session_id, sdp, profile) + else: + dialog.notification.show( + f"Invalid role received during setup: {setup_data}", + level="error" + ) return - for media_type, ice_data in candidates.items(): - for candidate in ice_data["candidates"]: - candidate_sdp = self.build_ice_candidate(candidate) - try: - sdp_mline_index = self.get_sdp_mline_index(media_type) - except Exception as e: - log.warning(e) - continue - ice_candidate = window.RTCIceCandidate.new({ - "candidate": candidate_sdp, - "sdpMLineIndex": sdp_mline_index - } - ) - await self._peer_connection.addIceCandidate(ice_candidate) - - def on_track(self, event): - """New track has been received from peer - - @param event: Event associated with the new track - """ - if event.streams and event.streams[0]: - remote_stream = event.streams[0] - document["remote_video"].srcObject = remote_stream - else: - if self.remote_stream is None: - self.remote_stream = window.MediaStream.new() - document["remote_video"].srcObject = self.remote_stream - self.remote_stream.addTrack(event.track) - - document["call_btn"].classList.add("is-hidden") - document["hangup_btn"].classList.remove("is-hidden") - - def on_negotiation_needed(self, event) -> None: - log.debug(f"on_negotiation_needed {event=}") - # TODO - - async def answer_call(self, offer_sdp: str, action_id: str): - """We respond to the call""" - log.debug("answering call") - await self._create_peer_connection() - - await self._peer_connection.setRemoteDescription({ - "type": "offer", - "sdp": offer_sdp - }) - await self.on_ice_candidates_new(self.candidates_buffer) - self.candidates_buffer.clear() - await self._get_user_media() - - # Gather local ICE candidates - local_ice_data = await self._gather_ice_candidates(False) - self.local_candidates = local_ice_data["candidates"] - - await bridge.action_launch( - action_id, - json.dumps({ - "sdp": self._peer_connection.localDescription.sdp, - }) - ) async def make_call(self, audio: bool = True, video: bool = True) -> None: """Start a WebRTC call @@ -445,69 +281,207 @@ @param audio: True if an audio flux is required @param video: True if a video flux is required """ - await self._create_peer_connection() - await self._get_user_media(audio, video) - await self._gather_ice_candidates(True) - callee_jid = document["callee_jid"].value + try: + callee_jid = JID(document["search"].value.strip()) + if not callee_jid.is_valid: + raise ValueError + except ValueError: + dialog.notification.show( + "Invalid identifier, please use a valid callee identifier", + level="error" + ) + return - call_data = { - "sdp": self._peer_connection.localDescription.sdp - } - log.info(f"calling {callee_jid!r}") - self.sid = await bridge.call_start( - callee_jid, - json.dumps(call_data) - ) - log.debug(f"Call SID: {self.sid}") + self._callee = callee_jid + await cache.fill_identities([callee_jid]) + self.status = "dialing" + call_avatar_elt = self.call_avatar_tpl.get_elt({ + "entity": str(callee_jid), + "identities": cache.identities, + }) + self.call_avatar_wrapper_elt.clear() + self.call_avatar_wrapper_elt <= call_avatar_elt - async def end_call(self) -> None: + + self.switch_mode("call") + await self.webrtc.make_call(callee_jid, audio, video) + + async def end_call(self, data: dict) -> None: """Stop streaming and clean instance""" - document["hangup_btn"].classList.add("is-hidden") - document["call_btn"].classList.remove("is-hidden") - if self._peer_connection is None: - log.debug("There is currently no call to end.") - else: - self._peer_connection.removeEventListener("track", self.on_track) - self._peer_connection.removeEventListener("negotiationneeded", self.on_negotiation_needed) - self._peer_connection.removeEventListener("icecandidate", self.on_ice_candidate) - self._peer_connection.removeEventListener("icegatheringstatechange", self.on_ice_gathering_state_change) + # if there is any ringing, we stop it + self.audio_player_elt.pause() + self.audio_player_elt.currentTime = 0 + + if self.incoming_call_dialog_elt is not None: + self.incoming_call_dialog_elt.cancel() + self.incoming_call_dialog_elt = None + + self.switch_mode("search") - local_video = document["local_video"] - remote_video = document["remote_video"] - if local_video.srcObject: - for track in local_video.srcObject.getTracks(): - track.stop() - if remote_video.srcObject: - for track in remote_video.srcObject.getTracks(): - track.stop() + if data.get("reason") == "busy": + assert self._callee is not None + peer_name = cache.identities[self._callee]["nicknames"][0] + dialog.notification.show( + f"{peer_name} can't answer your call", + level="info", + ) - self._peer_connection.close() - self.reset_instance() + await self.webrtc.end_call() - async def hand_up(self) -> None: + async def hang_up(self) -> None: """Terminate the call""" session_id = self.sid - await self.end_call() + if not session_id: + log.warning("Can't hand_up, not call in progress") + return + await self.end_call({"reason": "terminated"}) await bridge.call_end( session_id, "" ) + def _handle_animation_end( + self, + element, + remove = None, + add = None, + ): + """Return a handler that removes specified classes and the event handler. -webrtc_call = WebRTCCall() + @param element: The element to operate on. + @param remove: List of class names to remove from the element. + @param add: List of class names to add to the element. + """ + def handler(__, remove=remove, add=add): + log.info(f"animation end OK {element=}") + if add: + if isinstance(add, str): + add = [add] + element.classList.add(*add) + if remove: + if isinstance(remove, str): + remove = [remove] + element.classList.remove(*remove) + element.unbind('animationend', handler) + + return handler -document["call_btn"].bind( - "click", - lambda __: aio.run(webrtc_call.make_call()) -) -document["hangup_btn"].bind( - "click", - lambda __: aio.run(webrtc_call.hand_up()) -) + def switch_mode(self, mode: str) -> None: + """Handles the user interface changes""" + if mode == self.mode: + return + if mode == "call": + # Hide contacts with fade-out animation and bring up the call box + self.search_container_elt.classList.add("fade-out-y") + self.search_container_elt.bind( + 'animationend', + self._handle_animation_end( + self.search_container_elt, + remove="fade-out-y", + add="is-hidden" + ) + ) + self.call_container_elt.classList.remove("is-hidden") + self.call_container_elt.classList.add("slide-in") + self.call_container_elt.bind( + 'animationend', + self._handle_animation_end( + self.call_container_elt, + remove="slide-in" + ) + ) + self.mode = mode + elif mode == "search": + self.toggle_fullscreen(False) + self.search_container_elt.classList.add("fade-out-y", "animation-reverse") + self.search_container_elt.classList.remove("is-hidden") + self.search_container_elt.bind( + 'animationend', + self._handle_animation_end( + self.search_container_elt, + remove=["fade-out-y", "animation-reverse"], + ) + ) + self.call_container_elt.classList.add("slide-in", "animation-reverse") + self.call_container_elt.bind( + 'animationend', + self._handle_animation_end( + self.call_container_elt, + remove=["slide-in", "animation-reverse"], + add="is-hidden" + ) + ) + self.mode = mode + else: + log.error(f"Internal Error: Unknown call mode: {mode}") + + def toggle_fullscreen(self, fullscreen: bool|None = None): + """Toggle fullscreen mode for video elements. + + @param fullscreen: if set, determine the fullscreen state; otherwise, + the fullscreen mode will be toggled. + """ + do_fullscreen = ( + document.fullscreenElement is None if fullscreen is None else fullscreen + ) -bridge.register_signal("action_new", webrtc_call.on_action_new) -bridge.register_signal("call_accepted", webrtc_call._on_call_accepted) -bridge.register_signal("call_ended", webrtc_call._on_call_ended) -bridge.register_signal("ice_candidates_new", webrtc_call._on_ice_candidates_new) + try: + if do_fullscreen: + if document.fullscreenElement is None: + self.call_box_elt.requestFullscreen() + document["full_screen_btn"].classList.add("is-hidden") + document["exit_full_screen_btn"].classList.remove("is-hidden") + else: + if document.fullscreenElement is not None: + document.exitFullscreen() + document["full_screen_btn"].classList.remove("is-hidden") + document["exit_full_screen_btn"].classList.add("is-hidden") + + except Exception as e: + dialog.notification.show( + f"An error occurred while toggling fullscreen: {e}", + level="error" + ) + def toggle_audio_mute(self, evt): + is_muted = self.webrtc.toggle_audio_mute() + btn_elt = evt.currentTarget + if is_muted: + btn_elt.classList.remove("is-success") + btn_elt.classList.add("muted", "is-warning") + dialog.notification.show( + f"audio is now muted", + level="info", + delay=2, + ) + else: + btn_elt.classList.remove("muted", "is-warning") + btn_elt.classList.add("is-success") + + def toggle_video_mute(self, evt): + is_muted = self.webrtc.toggle_video_mute() + btn_elt = evt.currentTarget + if is_muted: + btn_elt.classList.remove("is-success") + btn_elt.classList.add("muted", "is-warning") + dialog.notification.show( + f"video is now muted", + level="info", + delay=2, + ) + else: + btn_elt.classList.remove("muted", "is-warning") + btn_elt.classList.add("is-success") + + def _on_entity_click(self, item: dict) -> None: + aio.run(self.on_entity_click(item)) + + async def on_entity_click(self, item: dict) -> None: + """Set entity JID to search bar, and start the call""" + document["search"].value = item["entity"] + + await self.make_call() + + +CallUI() loading.remove_loading_screen()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libervia/web/pages/calls/_browser/webrtc.py Wed Aug 09 00:22:18 2023 +0200 @@ -0,0 +1,489 @@ +import json +import re + +from bridge import AsyncBridge as Bridge +from browser import aio, console as log, document, timer, window +import errors +import jid + +log.warning = log.warn +profile = window.profile or "" +bridge = Bridge() +GATHER_TIMEOUT = 10000 + + +class WebRTC: + + def __init__(self): + self.reset_instance() + bridge.register_signal("ice_candidates_new", self._on_ice_candidates_new) + self.is_audio_muted = None + self.is_video_muted = None + self.local_video_elt = document["local_video"] + self.remote_video_elt = document["remote_video"] + + def reset_instance(self): + """Inits or resets the instance variables to their default state.""" + self._peer_connection = None + self._media_types = None + self._media_types_inv = None + self._callee = None + self.sid = None + self.local_candidates = None + self.remote_stream = None + self.candidates_buffer = { + "audio": {"candidates": []}, + "video": {"candidates": []}, + } + self.media_candidates = {} + self.candidates_gathered = aio.Future() + + @property + def media_types(self): + if self._media_types is None: + raise Exception("self._media_types should not be None!") + return self._media_types + + @media_types.setter + def media_types(self, new_media_types: dict) -> None: + self._media_types = new_media_types + self._media_types_inv = {v:k for k,v in new_media_types.items()} + + @property + def media_types_inv(self) -> dict: + if self._media_types_inv is None: + raise Exception("self._media_types_inv should not be None!") + return self._media_types_inv + + def get_sdp_mline_index(self, media_type): + """Gets the sdpMLineIndex for a given media type. + + @param media_type: The type of the media. + """ + for index, m_type in self.media_types.items(): + if m_type == media_type: + return index + raise ValueError(f"Media type '{media_type}' not found") + + def extract_pwd_ufrag(self, sdp): + """Retrieves ICE password and user fragment for SDP offer. + + @param sdp: The Session Description Protocol offer string. + """ + ufrag_line = re.search(r"ice-ufrag:(\S+)", sdp) + pwd_line = re.search(r"ice-pwd:(\S+)", sdp) + + if ufrag_line and pwd_line: + return ufrag_line.group(1), pwd_line.group(1) + else: + log.error(f"SDP with missing ice-ufrag or ice-pwd:\n{sdp}") + raise ValueError("Can't extract ice-ufrag and ice-pwd from SDP") + + def extract_fingerprint_data(self, sdp): + """Retrieves fingerprint data from an SDP offer. + + @param sdp: The Session Description Protocol offer string. + @return: A dictionary containing the fingerprint data. + """ + fingerprint_line = re.search(r"a=fingerprint:(\S+)\s+(\S+)", sdp) + if fingerprint_line: + algorithm, fingerprint = fingerprint_line.groups() + fingerprint_data = { + "hash": algorithm, + "fingerprint": fingerprint + } + + setup_line = re.search(r"a=setup:(\S+)", sdp) + if setup_line: + setup = setup_line.group(1) + fingerprint_data["setup"] = setup + + return fingerprint_data + else: + raise ValueError("fingerprint should not be missing") + + def parse_ice_candidate(self, candidate_string): + """Parses the ice candidate string. + + @param candidate_string: The ice candidate string to be parsed. + """ + pattern = re.compile( + r"candidate:(?P<foundation>\S+) (?P<component_id>\d+) (?P<transport>\S+) " + r"(?P<priority>\d+) (?P<address>\S+) (?P<port>\d+) typ " + r"(?P<type>\S+)(?: raddr (?P<rel_addr>\S+) rport " + r"(?P<rel_port>\d+))?(?: generation (?P<generation>\d+))?" + ) + match = pattern.match(candidate_string) + if match: + candidate_dict = match.groupdict() + + # Apply the correct types to the dictionary values + candidate_dict["component_id"] = int(candidate_dict["component_id"]) + candidate_dict["priority"] = int(candidate_dict["priority"]) + candidate_dict["port"] = int(candidate_dict["port"]) + + if candidate_dict["rel_port"]: + candidate_dict["rel_port"] = int(candidate_dict["rel_port"]) + + if candidate_dict["generation"]: + candidate_dict["generation"] = candidate_dict["generation"] + + # Remove None values + return {k: v for k, v in candidate_dict.items() if v is not None} + else: + log.warning(f"can't parse candidate: {candidate_string!r}") + return None + + def build_ice_candidate(self, parsed_candidate): + """Builds ICE candidate + + @param parsed_candidate: Dictionary containing parsed ICE candidate + """ + base_format = ( + "candidate:{foundation} {component_id} {transport} {priority} " + "{address} {port} typ {type}" + ) + + if ((parsed_candidate.get('rel_addr') + and parsed_candidate.get('rel_port'))): + base_format += " raddr {rel_addr} rport {rel_port}" + + if parsed_candidate.get('generation'): + base_format += " generation {generation}" + + return base_format.format(**parsed_candidate) + + def on_ice_candidate(self, event): + """Handles ICE candidate event + + @param event: Event containing the ICE candidate + """ + log.debug(f"on ice candidate {event.candidate=}") + if event.candidate and event.candidate.candidate: + window.last_event = event + parsed_candidate = self.parse_ice_candidate(event.candidate.candidate) + if parsed_candidate is None: + return + try: + media_type = self.media_types[event.candidate.sdpMLineIndex] + except (TypeError, IndexError): + log.error( + f"Can't find media type.\n{event.candidate=}\n{self._media_types=}" + ) + return + self.media_candidates.setdefault(media_type, []).append(parsed_candidate) + log.debug(f"ICE candidate [{media_type}]: {event.candidate.candidate}") + else: + log.debug("All ICE candidates gathered") + + def _set_media_types(self, offer): + """Sets media types from offer SDP + + @param offer: RTC session description containing the offer + """ + sdp_lines = offer.sdp.splitlines() + media_types = {} + mline_index = 0 + + for line in sdp_lines: + if line.startswith("m="): + media_types[mline_index] = line[2:line.find(" ")] + mline_index += 1 + + self.media_types = media_types + + def on_ice_gathering_state_change(self, event): + """Handles ICE gathering state change + + @param event: Event containing the ICE gathering state change + """ + connection = event.target + log.debug(f"on_ice_gathering_state_change {connection.iceGatheringState=}") + if connection.iceGatheringState == "complete": + log.info("ICE candidates gathering done") + self.candidates_gathered.set_result(None) + + async def _create_peer_connection( + self, + ): + """Creates peer connection""" + if self._peer_connection is not None: + raise Exception("create_peer_connection can't be called twice!") + + external_disco = json.loads(await bridge.external_disco_get("")) + ice_servers = [] + + for server in external_disco: + ice_server = {} + if server["type"] == "stun": + ice_server["urls"] = f"stun:{server['host']}:{server['port']}" + elif server["type"] == "turn": + ice_server["urls"] = ( + f"turn:{server['host']}:{server['port']}?transport={server['transport']}" + ) + ice_server["username"] = server["username"] + ice_server["credential"] = server["password"] + ice_servers.append(ice_server) + + rtc_configuration = {"iceServers": ice_servers} + + peer_connection = window.RTCPeerConnection.new(rtc_configuration) + peer_connection.addEventListener("track", self.on_track) + peer_connection.addEventListener("negotiationneeded", self.on_negotiation_needed) + peer_connection.addEventListener("icecandidate", self.on_ice_candidate) + peer_connection.addEventListener("icegatheringstatechange", self.on_ice_gathering_state_change) + + self._peer_connection = peer_connection + window.pc = self._peer_connection + + async def _get_user_media( + self, + audio: bool = True, + video: bool = True + ): + """Gets user media + + @param audio: True if an audio flux is required + @param video: True if a video flux is required + """ + media_constraints = {'audio': audio, 'video': video} + local_stream = await window.navigator.mediaDevices.getUserMedia(media_constraints) + self.local_video_elt.srcObject = local_stream + + for track in local_stream.getTracks(): + self._peer_connection.addTrack(track) + + async def _gather_ice_candidates(self, is_initiator: bool, remote_candidates=None): + """Get ICE candidates and wait to have them all before returning them + + @param is_initiator: Boolean indicating if the user is the initiator of the connection + @param remote_candidates: Remote ICE candidates, if any + """ + if self._peer_connection is None: + raise Exception("The peer connection must be created before gathering ICE candidates!") + + self.media_candidates.clear() + gather_timeout = timer.set_timeout( + lambda: self.candidates_gathered.set_exception( + errors.TimeoutError("ICE gathering time out") + ), + GATHER_TIMEOUT + ) + + if is_initiator: + offer = await self._peer_connection.createOffer() + self._set_media_types(offer) + await self._peer_connection.setLocalDescription(offer) + else: + answer = await self._peer_connection.createAnswer() + self._set_media_types(answer) + await self._peer_connection.setLocalDescription(answer) + + if not is_initiator: + log.debug(self._peer_connection.localDescription.sdp) + await self.candidates_gathered + log.debug(self._peer_connection.localDescription.sdp) + timer.clear_timeout(gather_timeout) + ufrag, pwd = self.extract_pwd_ufrag(self._peer_connection.localDescription.sdp) + return { + "ufrag": ufrag, + "pwd": pwd, + "candidates": self.media_candidates, + } + + async def accept_call(self, session_id: str, sdp: str, profile: str) -> None: + """Call has been accepted, connection can be established + + @param session_id: Session identifier + @param sdp: Session Description Protocol data + @param profile: Profile associated + """ + await self._peer_connection.setRemoteDescription({ + "type": "answer", + "sdp": sdp + }) + await self.on_ice_candidates_new(self.candidates_buffer) + self.candidates_buffer.clear() + + def _on_ice_candidates_new(self, sid: str, candidates_s: str, profile: str) -> None: + """Called when new ICE candidates are received + + @param sid: Session identifier + @param candidates_s: ICE candidates serialized + @param profile: Profile associated with the action + """ + if sid != self.sid: + log.debug( + f"ignoring peer ice candidates for {sid=} ({self.sid=})." + ) + return + candidates = json.loads(candidates_s) + aio.run(self.on_ice_candidates_new(candidates)) + + async def on_ice_candidates_new(self, candidates: dict) -> None: + """Called when new ICE canidates are received from peer + + @param candidates: Dictionary containing new ICE candidates + """ + log.debug(f"new peer candidates received: {candidates}") + if ( + self._peer_connection is None + or self._peer_connection.remoteDescription is None + ): + for media_type in ("audio", "video"): + media_candidates = candidates.get(media_type) + if media_candidates: + buffer = self.candidates_buffer[media_type] + buffer["candidates"].extend(media_candidates["candidates"]) + return + for media_type, ice_data in candidates.items(): + for candidate in ice_data["candidates"]: + candidate_sdp = self.build_ice_candidate(candidate) + try: + sdp_mline_index = self.get_sdp_mline_index(media_type) + except Exception as e: + log.warning(e) + continue + ice_candidate = window.RTCIceCandidate.new({ + "candidate": candidate_sdp, + "sdpMLineIndex": sdp_mline_index + } + ) + await self._peer_connection.addIceCandidate(ice_candidate) + + def on_track(self, event): + """New track has been received from peer + + @param event: Event associated with the new track + """ + if event.streams and event.streams[0]: + remote_stream = event.streams[0] + self.remote_video_elt.srcObject = remote_stream + else: + if self.remote_stream is None: + self.remote_stream = window.MediaStream.new() + self.remote_video_elt.srcObject = self.remote_stream + self.remote_stream.addTrack(event.track) + + def on_negotiation_needed(self, event) -> None: + log.debug(f"on_negotiation_needed {event=}") + # TODO + + async def answer_call(self, sid: str, offer_sdp: str, profile: str): + """We respond to the call""" + log.debug("answering call") + if sid != self.sid: + raise Exception( + f"Internal Error: unexpected sid: {sid=} {self.sid=}" + ) + await self._create_peer_connection() + + await self._peer_connection.setRemoteDescription({ + "type": "offer", + "sdp": offer_sdp + }) + await self.on_ice_candidates_new(self.candidates_buffer) + self.candidates_buffer.clear() + await self._get_user_media() + + # Gather local ICE candidates + local_ice_data = await self._gather_ice_candidates(False) + self.local_candidates = local_ice_data["candidates"] + + await bridge.call_answer_sdp(sid, self._peer_connection.localDescription.sdp) + + async def make_call( + self, + callee_jid: jid.JID, + audio: bool = True, + video: bool = True + ) -> None: + """Start a WebRTC call + + @param audio: True if an audio flux is required + @param video: True if a video flux is required + """ + await self._create_peer_connection() + await self._get_user_media(audio, video) + await self._gather_ice_candidates(True) + + call_data = { + "sdp": self._peer_connection.localDescription.sdp + } + log.info(f"calling {callee_jid!r}") + self.sid = await bridge.call_start( + str(callee_jid), + json.dumps(call_data) + ) + log.debug(f"Call SID: {self.sid}") + + async def end_call(self) -> None: + """Stop streaming and clean instance""" + if self._peer_connection is None: + log.debug("There is currently no call to end.") + else: + self._peer_connection.removeEventListener("track", self.on_track) + self._peer_connection.removeEventListener("negotiationneeded", self.on_negotiation_needed) + self._peer_connection.removeEventListener("icecandidate", self.on_ice_candidate) + self._peer_connection.removeEventListener("icegatheringstatechange", self.on_ice_gathering_state_change) + + # Base64 encoded 1x1 black pixel image + # this is a trick to reset the image displayed, so we don't see last image of + # last stream + black_image_data = ( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0" + "lEQVR42mP8/wcAAwAB/uzNq7sAAAAASUVORK5CYII=" + ) + + local_video = self.local_video_elt + remote_video = self.remote_video_elt + if local_video.srcObject: + for track in local_video.srcObject.getTracks(): + track.stop() + local_video.src = black_image_data + + if remote_video.srcObject: + for track in remote_video.srcObject.getTracks(): + track.stop() + remote_video.src = black_image_data + + self._peer_connection.close() + self.reset_instance() + + def toggle_media_mute(self, media_type: str) -> bool: + """Toggle mute/unmute for media tracks. + + @param media_type: 'audio' or 'video'. Determines which media tracks + to process. + """ + assert media_type in ("audio", "video"), "Invalid media type" + + local_video = self.local_video_elt + is_muted_attr = f"is_{media_type}_muted" + + if local_video.srcObject: + track_getter = getattr(local_video.srcObject, f"get{media_type.capitalize()}Tracks") + for track in track_getter(): + track.enabled = not track.enabled + setattr(self, is_muted_attr, not track.enabled) + + media_name = self.media_types_inv.get(media_type) + if media_name is not None: + extra = {"name": str(media_name)} + aio.run( + bridge.call_info( + self.sid, + "mute" if getattr(self, is_muted_attr) else "unmute", + json.dumps(extra), + ) + ) + + return getattr(self, is_muted_attr) + + def toggle_audio_mute(self) -> bool: + """Toggle mute/unmute for audio tracks.""" + return self.toggle_media_mute("audio") + + def toggle_video_mute(self) -> bool: + """Toggle mute/unmute for video tracks.""" + return self.toggle_media_mute("video")
--- a/libervia/web/server/restricted_bridge.py Wed Aug 09 00:22:16 2023 +0200 +++ b/libervia/web/server/restricted_bridge.py Wed Aug 09 00:22:18 2023 +0200 @@ -53,6 +53,22 @@ "call_start", entity, call_data_s, profile ) + async def call_answer_sdp( + self, session_id: str, answer_sdp: str, profile: str + ) -> None: + self.no_service_profile(profile) + return await self.host.bridge_call( + "call_answer_sdp", session_id, answer_sdp, profile + ) + + async def call_info( + self, session_id: str, info_type: str, extra_s: str, profile: str + ) -> None: + self.no_service_profile(profile) + return await self.host.bridge_call( + "call_info", session_id, info_type, extra_s, profile + ) + async def call_end(self, session_id: str, call_data: str, profile: str) -> None: self.no_service_profile(profile) return await self.host.bridge_call( @@ -156,6 +172,14 @@ return await self.host.bridge_call( "interest_retract", service_jid, item_id, profile) + async def jingle_terminate( + self, session_id: str, reason: str, reason_txt: str, profile: str + ) -> None: + self.no_service_profile(profile) + return await self.host.bridge_call( + "jingle_terminate", session_id, reason, reason_txt, profile + ) + async def ps_invite( self, invitee_jid_s, service_s, node, item_id, name, extra_s, profile ):
--- a/libervia/web/server/server.py Wed Aug 09 00:22:16 2023 +0200 +++ b/libervia/web/server/server.py Wed Aug 09 00:22:18 2023 +0200 @@ -574,7 +574,7 @@ "message_new", partial(self.on_signal, "message_new") ) self.bridge.register_signal( - "call_accepted", partial(self.on_signal, "call_accepted"), "plugin" + "call_setup", partial(self.on_signal, "call_setup"), "plugin" ) self.bridge.register_signal( "call_ended", partial(self.on_signal, "call_ended"), "plugin"