Mercurial > libervia-web
view libervia/web/pages/calls/_browser/webrtc.py @ 1549:e47c24204449
browser (calls): update call to handle search, control buttons, and better UI/UX:
- adapt to backend changes
- UI and WebRTC parts are not separated
- users can now be searched
- add mute/fullscreen buttons
- ring
- cancellable dialog when a call is received
- status of the call
- animations
- various UI/UX improvments
rel 423
author | Goffi <goffi@goffi.org> |
---|---|
date | Wed, 09 Aug 2023 00:22:18 +0200 |
parents | libervia/web/pages/calls/_browser/__init__.py@eb00d593801d |
children | 83c2a6faa2ae |
line wrap: on
line source
import json import re from bridge import AsyncBridge as Bridge from browser import aio, console as log, document, timer, window import errors import jid log.warning = log.warn profile = window.profile or "" bridge = Bridge() GATHER_TIMEOUT = 10000 class WebRTC: def __init__(self): self.reset_instance() bridge.register_signal("ice_candidates_new", self._on_ice_candidates_new) self.is_audio_muted = None self.is_video_muted = None self.local_video_elt = document["local_video"] self.remote_video_elt = document["remote_video"] def reset_instance(self): """Inits or resets the instance variables to their default state.""" self._peer_connection = None self._media_types = None self._media_types_inv = None self._callee = None self.sid = None self.local_candidates = None self.remote_stream = None self.candidates_buffer = { "audio": {"candidates": []}, "video": {"candidates": []}, } self.media_candidates = {} self.candidates_gathered = aio.Future() @property def media_types(self): if self._media_types is None: raise Exception("self._media_types should not be None!") return self._media_types @media_types.setter def media_types(self, new_media_types: dict) -> None: self._media_types = new_media_types self._media_types_inv = {v:k for k,v in new_media_types.items()} @property def media_types_inv(self) -> dict: if self._media_types_inv is None: raise Exception("self._media_types_inv should not be None!") return self._media_types_inv def get_sdp_mline_index(self, media_type): """Gets the sdpMLineIndex for a given media type. @param media_type: The type of the media. """ for index, m_type in self.media_types.items(): if m_type == media_type: return index raise ValueError(f"Media type '{media_type}' not found") def extract_pwd_ufrag(self, sdp): """Retrieves ICE password and user fragment for SDP offer. @param sdp: The Session Description Protocol offer string. """ ufrag_line = re.search(r"ice-ufrag:(\S+)", sdp) pwd_line = re.search(r"ice-pwd:(\S+)", sdp) if ufrag_line and pwd_line: return ufrag_line.group(1), pwd_line.group(1) else: log.error(f"SDP with missing ice-ufrag or ice-pwd:\n{sdp}") raise ValueError("Can't extract ice-ufrag and ice-pwd from SDP") def extract_fingerprint_data(self, sdp): """Retrieves fingerprint data from an SDP offer. @param sdp: The Session Description Protocol offer string. @return: A dictionary containing the fingerprint data. """ fingerprint_line = re.search(r"a=fingerprint:(\S+)\s+(\S+)", sdp) if fingerprint_line: algorithm, fingerprint = fingerprint_line.groups() fingerprint_data = { "hash": algorithm, "fingerprint": fingerprint } setup_line = re.search(r"a=setup:(\S+)", sdp) if setup_line: setup = setup_line.group(1) fingerprint_data["setup"] = setup return fingerprint_data else: raise ValueError("fingerprint should not be missing") def parse_ice_candidate(self, candidate_string): """Parses the ice candidate string. @param candidate_string: The ice candidate string to be parsed. """ pattern = re.compile( r"candidate:(?P<foundation>\S+) (?P<component_id>\d+) (?P<transport>\S+) " r"(?P<priority>\d+) (?P<address>\S+) (?P<port>\d+) typ " r"(?P<type>\S+)(?: raddr (?P<rel_addr>\S+) rport " r"(?P<rel_port>\d+))?(?: generation (?P<generation>\d+))?" ) match = pattern.match(candidate_string) if match: candidate_dict = match.groupdict() # Apply the correct types to the dictionary values candidate_dict["component_id"] = int(candidate_dict["component_id"]) candidate_dict["priority"] = int(candidate_dict["priority"]) candidate_dict["port"] = int(candidate_dict["port"]) if candidate_dict["rel_port"]: candidate_dict["rel_port"] = int(candidate_dict["rel_port"]) if candidate_dict["generation"]: candidate_dict["generation"] = candidate_dict["generation"] # Remove None values return {k: v for k, v in candidate_dict.items() if v is not None} else: log.warning(f"can't parse candidate: {candidate_string!r}") return None def build_ice_candidate(self, parsed_candidate): """Builds ICE candidate @param parsed_candidate: Dictionary containing parsed ICE candidate """ base_format = ( "candidate:{foundation} {component_id} {transport} {priority} " "{address} {port} typ {type}" ) if ((parsed_candidate.get('rel_addr') and parsed_candidate.get('rel_port'))): base_format += " raddr {rel_addr} rport {rel_port}" if parsed_candidate.get('generation'): base_format += " generation {generation}" return base_format.format(**parsed_candidate) def on_ice_candidate(self, event): """Handles ICE candidate event @param event: Event containing the ICE candidate """ log.debug(f"on ice candidate {event.candidate=}") if event.candidate and event.candidate.candidate: window.last_event = event parsed_candidate = self.parse_ice_candidate(event.candidate.candidate) if parsed_candidate is None: return try: media_type = self.media_types[event.candidate.sdpMLineIndex] except (TypeError, IndexError): log.error( f"Can't find media type.\n{event.candidate=}\n{self._media_types=}" ) return self.media_candidates.setdefault(media_type, []).append(parsed_candidate) log.debug(f"ICE candidate [{media_type}]: {event.candidate.candidate}") else: log.debug("All ICE candidates gathered") def _set_media_types(self, offer): """Sets media types from offer SDP @param offer: RTC session description containing the offer """ sdp_lines = offer.sdp.splitlines() media_types = {} mline_index = 0 for line in sdp_lines: if line.startswith("m="): media_types[mline_index] = line[2:line.find(" ")] mline_index += 1 self.media_types = media_types def on_ice_gathering_state_change(self, event): """Handles ICE gathering state change @param event: Event containing the ICE gathering state change """ connection = event.target log.debug(f"on_ice_gathering_state_change {connection.iceGatheringState=}") if connection.iceGatheringState == "complete": log.info("ICE candidates gathering done") self.candidates_gathered.set_result(None) async def _create_peer_connection( self, ): """Creates peer connection""" if self._peer_connection is not None: raise Exception("create_peer_connection can't be called twice!") external_disco = json.loads(await bridge.external_disco_get("")) ice_servers = [] for server in external_disco: ice_server = {} if server["type"] == "stun": ice_server["urls"] = f"stun:{server['host']}:{server['port']}" elif server["type"] == "turn": ice_server["urls"] = ( f"turn:{server['host']}:{server['port']}?transport={server['transport']}" ) ice_server["username"] = server["username"] ice_server["credential"] = server["password"] ice_servers.append(ice_server) rtc_configuration = {"iceServers": ice_servers} peer_connection = window.RTCPeerConnection.new(rtc_configuration) peer_connection.addEventListener("track", self.on_track) peer_connection.addEventListener("negotiationneeded", self.on_negotiation_needed) peer_connection.addEventListener("icecandidate", self.on_ice_candidate) peer_connection.addEventListener("icegatheringstatechange", self.on_ice_gathering_state_change) self._peer_connection = peer_connection window.pc = self._peer_connection async def _get_user_media( self, audio: bool = True, video: bool = True ): """Gets user media @param audio: True if an audio flux is required @param video: True if a video flux is required """ media_constraints = {'audio': audio, 'video': video} local_stream = await window.navigator.mediaDevices.getUserMedia(media_constraints) self.local_video_elt.srcObject = local_stream for track in local_stream.getTracks(): self._peer_connection.addTrack(track) async def _gather_ice_candidates(self, is_initiator: bool, remote_candidates=None): """Get ICE candidates and wait to have them all before returning them @param is_initiator: Boolean indicating if the user is the initiator of the connection @param remote_candidates: Remote ICE candidates, if any """ if self._peer_connection is None: raise Exception("The peer connection must be created before gathering ICE candidates!") self.media_candidates.clear() gather_timeout = timer.set_timeout( lambda: self.candidates_gathered.set_exception( errors.TimeoutError("ICE gathering time out") ), GATHER_TIMEOUT ) if is_initiator: offer = await self._peer_connection.createOffer() self._set_media_types(offer) await self._peer_connection.setLocalDescription(offer) else: answer = await self._peer_connection.createAnswer() self._set_media_types(answer) await self._peer_connection.setLocalDescription(answer) if not is_initiator: log.debug(self._peer_connection.localDescription.sdp) await self.candidates_gathered log.debug(self._peer_connection.localDescription.sdp) timer.clear_timeout(gather_timeout) ufrag, pwd = self.extract_pwd_ufrag(self._peer_connection.localDescription.sdp) return { "ufrag": ufrag, "pwd": pwd, "candidates": self.media_candidates, } async def accept_call(self, session_id: str, sdp: str, profile: str) -> None: """Call has been accepted, connection can be established @param session_id: Session identifier @param sdp: Session Description Protocol data @param profile: Profile associated """ await self._peer_connection.setRemoteDescription({ "type": "answer", "sdp": sdp }) await self.on_ice_candidates_new(self.candidates_buffer) self.candidates_buffer.clear() def _on_ice_candidates_new(self, sid: str, candidates_s: str, profile: str) -> None: """Called when new ICE candidates are received @param sid: Session identifier @param candidates_s: ICE candidates serialized @param profile: Profile associated with the action """ if sid != self.sid: log.debug( f"ignoring peer ice candidates for {sid=} ({self.sid=})." ) return candidates = json.loads(candidates_s) aio.run(self.on_ice_candidates_new(candidates)) async def on_ice_candidates_new(self, candidates: dict) -> None: """Called when new ICE canidates are received from peer @param candidates: Dictionary containing new ICE candidates """ log.debug(f"new peer candidates received: {candidates}") if ( self._peer_connection is None or self._peer_connection.remoteDescription is None ): for media_type in ("audio", "video"): media_candidates = candidates.get(media_type) if media_candidates: buffer = self.candidates_buffer[media_type] buffer["candidates"].extend(media_candidates["candidates"]) return for media_type, ice_data in candidates.items(): for candidate in ice_data["candidates"]: candidate_sdp = self.build_ice_candidate(candidate) try: sdp_mline_index = self.get_sdp_mline_index(media_type) except Exception as e: log.warning(e) continue ice_candidate = window.RTCIceCandidate.new({ "candidate": candidate_sdp, "sdpMLineIndex": sdp_mline_index } ) await self._peer_connection.addIceCandidate(ice_candidate) def on_track(self, event): """New track has been received from peer @param event: Event associated with the new track """ if event.streams and event.streams[0]: remote_stream = event.streams[0] self.remote_video_elt.srcObject = remote_stream else: if self.remote_stream is None: self.remote_stream = window.MediaStream.new() self.remote_video_elt.srcObject = self.remote_stream self.remote_stream.addTrack(event.track) def on_negotiation_needed(self, event) -> None: log.debug(f"on_negotiation_needed {event=}") # TODO async def answer_call(self, sid: str, offer_sdp: str, profile: str): """We respond to the call""" log.debug("answering call") if sid != self.sid: raise Exception( f"Internal Error: unexpected sid: {sid=} {self.sid=}" ) await self._create_peer_connection() await self._peer_connection.setRemoteDescription({ "type": "offer", "sdp": offer_sdp }) await self.on_ice_candidates_new(self.candidates_buffer) self.candidates_buffer.clear() await self._get_user_media() # Gather local ICE candidates local_ice_data = await self._gather_ice_candidates(False) self.local_candidates = local_ice_data["candidates"] await bridge.call_answer_sdp(sid, self._peer_connection.localDescription.sdp) async def make_call( self, callee_jid: jid.JID, audio: bool = True, video: bool = True ) -> None: """Start a WebRTC call @param audio: True if an audio flux is required @param video: True if a video flux is required """ await self._create_peer_connection() await self._get_user_media(audio, video) await self._gather_ice_candidates(True) call_data = { "sdp": self._peer_connection.localDescription.sdp } log.info(f"calling {callee_jid!r}") self.sid = await bridge.call_start( str(callee_jid), json.dumps(call_data) ) log.debug(f"Call SID: {self.sid}") async def end_call(self) -> None: """Stop streaming and clean instance""" if self._peer_connection is None: log.debug("There is currently no call to end.") else: self._peer_connection.removeEventListener("track", self.on_track) self._peer_connection.removeEventListener("negotiationneeded", self.on_negotiation_needed) self._peer_connection.removeEventListener("icecandidate", self.on_ice_candidate) self._peer_connection.removeEventListener("icegatheringstatechange", self.on_ice_gathering_state_change) # Base64 encoded 1x1 black pixel image # this is a trick to reset the image displayed, so we don't see last image of # last stream black_image_data = ( "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0" "lEQVR42mP8/wcAAwAB/uzNq7sAAAAASUVORK5CYII=" ) local_video = self.local_video_elt remote_video = self.remote_video_elt if local_video.srcObject: for track in local_video.srcObject.getTracks(): track.stop() local_video.src = black_image_data if remote_video.srcObject: for track in remote_video.srcObject.getTracks(): track.stop() remote_video.src = black_image_data self._peer_connection.close() self.reset_instance() def toggle_media_mute(self, media_type: str) -> bool: """Toggle mute/unmute for media tracks. @param media_type: 'audio' or 'video'. Determines which media tracks to process. """ assert media_type in ("audio", "video"), "Invalid media type" local_video = self.local_video_elt is_muted_attr = f"is_{media_type}_muted" if local_video.srcObject: track_getter = getattr(local_video.srcObject, f"get{media_type.capitalize()}Tracks") for track in track_getter(): track.enabled = not track.enabled setattr(self, is_muted_attr, not track.enabled) media_name = self.media_types_inv.get(media_type) if media_name is not None: extra = {"name": str(media_name)} aio.run( bridge.call_info( self.sid, "mute" if getattr(self, is_muted_attr) else "unmute", json.dumps(extra), ) ) return getattr(self, is_muted_attr) def toggle_audio_mute(self) -> bool: """Toggle mute/unmute for audio tracks.""" return self.toggle_media_mute("audio") def toggle_video_mute(self) -> bool: """Toggle mute/unmute for video tracks.""" return self.toggle_media_mute("video")