Mercurial > libervia-web
changeset 1600:0a4433a343a3
browser (calls): implement WebRTC file sharing:
- Send file through WebRTC when the new `file` button is used during a call.
- Show a confirmation dialog and download file sent by WebRTC.
rel 442
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Sat, 06 Apr 2024 13:06:17 +0200 |
| parents | 197350e8bf3b |
| children | d07838fc9d99 |
| files | libervia/web/pages/_browser/dialog.py libervia/web/pages/calls/_browser/__init__.py libervia/web/pages/calls/_browser/webrtc.py libervia/web/server/restricted_bridge.py |
| diffstat | 4 files changed, 414 insertions(+), 65 deletions(-) [+] |
line wrap: on
line diff
--- a/libervia/web/pages/_browser/dialog.py Tue Mar 05 16:40:25 2024 +0100 +++ b/libervia/web/pages/_browser/dialog.py Sat Apr 06 13:06:17 2024 +0200 @@ -9,6 +9,11 @@ class CancelError(Exception): """Dialog is cancelled""" + def __init__(self, reason: str = "", text: str = "") -> None: + self.reason = reason + self.text = text + super().__init__(text) + class Confirm: @@ -34,7 +39,7 @@ def _default_cancel_cb(self, evt, notif_elt): notif_elt.remove() - def cancel(self): + def cancel(self, reason: str = "", text: str = ""): """Cancel the dialog, without calling any callback will raise a CancelError @@ -45,7 +50,7 @@ self._notif_elt.remove() self._notif_elt = None if self._reject_cb is not None: - self._reject_cb(CancelError) + self._reject_cb(CancelError(reason, text)) else: log.warning("no reject callback set") self.reset()
--- a/libervia/web/pages/calls/_browser/__init__.py Tue Mar 05 16:40:25 2024 +0100 +++ b/libervia/web/pages/calls/_browser/__init__.py Sat Apr 06 13:06:17 2024 +0200 @@ -4,6 +4,7 @@ from browser import aio, console as log, document, window from cache import cache import dialog +from javascript import JSObject from jid import JID from jid_search import JidSearch import loading @@ -42,9 +43,11 @@ on_video_devices=self.on_video_devices, on_reset_cb=self.on_reset_cb, ) + # mapping of file sending + self.files_webrtc: list[dict] = [] self.mode = "search" self._status = None - self._callee = None + self._callee: JID|None = None self.contacts_elt = document["contacts"] self.search_container_elt = document["search_container"] self.call_container_elt = document["call_container"] @@ -82,6 +85,8 @@ else: self.share_desktop_col_elt.classList.add("is-hidden") document["switch_camera_btn"].bind("click", self.on_switch_camera) + document["send_file_btn"].bind("click", self.on_send_file) + document["send_file_input"].bind("change", self._on_send_input_change) # search self.search_elt = document["search"] @@ -185,13 +190,128 @@ @param profile: Profile associated with the action """ action_data = json.loads(action_data_s) - if action_data.get("type") != "call": + if action_data.get("type") == "confirm" and action_data.get("subtype") == "file": + aio.run(self.on_file_preflight(action_data, action_id)) + elif action_data.get("type") == "file": + aio.run(self.on_file_proposal(action_data, action_id)) + elif action_data.get("type") != "call": return - aio.run(self.on_action_new(action_data, action_id)) + else: + aio.run(self.on_action_new(action_data, action_id)) + + def get_human_size(self, size: int|float) -> str: + """Return size in human-friendly size using SI units""" + units = ["o","Kio","Mio","Gio"] + for idx, unit in enumerate(units): + if size < 1024.0 or idx == len(units)-1: + return f"{size:.2f}{unit}" + size /= 1024.0 + raise Exception("Internal Error: this line should never be reached.") + + async def request_file_permission(self, action_data: dict) -> bool: + """Request permission to download a file.""" + peer_jid = JID(action_data["from_jid"]).bare + await cache.fill_identities([peer_jid]) + identity = cache.identities[peer_jid] + peer_name = identity["nicknames"][0] + + file_data = action_data.get("file_data", {}) + + file_name = file_data.get('name') + file_size = file_data.get('size') + + if file_name: + file_name_msg = 'wants to send you the file "{file_name}"'.format( + file_name=file_name + ) + else: + file_name_msg = 'wants to send you an unnamed file' + + if file_size is not None: + file_size_msg = "which has a size of {file_size_human}".format( + file_size_human=self.get_human_size(file_size) + ) + else: + file_size_msg = "which has an unknown size" + + file_description = file_data.get('desc') + if file_description: + description_msg = " Description: {}.".format(file_description) + else: + description_msg = "" + + file_data = action_data.get("file_data", {}) + + file_accept_dlg = dialog.Confirm( + "{peer_name} ({peer_jid}) {file_name_msg} {file_size_msg}.{description_msg} Do you " + "accept?".format( + peer_name=peer_name, + peer_jid=peer_jid, + file_name_msg=file_name_msg, + file_size_msg=file_size_msg, + description_msg=description_msg + ), + ok_label="Download", + cancel_label="Reject" + ) + return await file_accept_dlg.ashow() + + async def on_file_preflight(self, action_data: dict, action_id: str) -> None: + """Handle a file preflight (proposal made to all devices).""" + # FIXME: temporarily done in call page, will be moved to notifications handler to + # make it work anywhere. + accepted = await self.request_file_permission(action_data) + + await bridge.action_launch( + action_id, json.dumps({"answer": str(accepted).lower()}) + ) + + async def on_file_proposal(self, action_data: dict, action_id: str) -> None: + """Handle a file proposal. + + This is a proposal made specifically to this device, a opposed to + ``on_file_preflight``. File may already have been accepted during preflight. + """ + # FIXME: as for on_file_preflight, this will be moved to notification handler. + if not action_data.get("webrtc", False): + peer_jid = JID(action_data["from_jid"]).bare + # We try to do a not-too-technical warning about webrtc not being supported. + dialog.notification.show( + f"A file sending from {peer_jid} can't be accepted because it is not " + "compatible with web browser direct transfer (WebRTC).", + level="warning", + ) + # We don't explicitly refuse the file proposal, because it may be accepted and + # supported by other frontends. + # TODO: Check if any other frontend is connected for this profile, and refuse + # the file if none is. + return + if action_data.get("file_accepted", False): + # File proposal has already been accepted in preflight. + accepted = True + else: + accepted = await self.request_file_permission(action_data) + + if accepted: + sid = action_data["session_id"] + webrtc = WebRTC( + file_only=True, + extra_data={"file_data": action_data.get("file_data", {})} + ) + webrtc.sid = sid + self.files_webrtc.append({ + "webrtc": webrtc, + }) + + await bridge.action_launch( + action_id, json.dumps({"answer": str(accepted).lower()}) + ) async def on_action_new(self, action_data: dict, action_id: str) -> None: peer_jid = JID(action_data["from_jid"]).bare - log.info(f"{peer_jid} wants to start a call ({action_data['sub_type']})") + call_type = action_data["sub_type"] + call_emoji = "📹" if call_type == VIDEO else "📞" + log.info(f"{peer_jid} wants to start a call ({call_type}).") if self.sid is not None: log.warning( f"already in a call ({self.sid}), can't receive a new call from " @@ -210,14 +330,30 @@ # and ask user if we take the call try: self.incoming_call_dialog_elt = dialog.Confirm( - f"{peer_name} is calling you.", ok_label="Answer", cancel_label="Reject" + f"{peer_name} is calling you ({call_emoji}{call_type}).", ok_label="Answer", cancel_label="Reject" ) accepted = await self.incoming_call_dialog_elt.ashow() - except dialog.CancelError: + except dialog.CancelError as e: log.info("Call has been cancelled") self.incoming_call_dialog_elt = None self.sid = None - dialog.notification.show(f"{peer_name} has cancelled the call", level="info") + match e.reason: + case "busy": + dialog.notification.show( + f"{peer_name} can't answer your call", + level="info", + ) + case "taken_by_other_device": + device = e.text + dialog.notification.show( + f"The call has been taken on another device ({device}).", + level="info", + ) + case _: + dialog.notification.show( + f"{peer_name} has cancelled the call", + level="info" + ) return self.incoming_call_dialog_elt = None @@ -230,6 +366,7 @@ log.debug(f"Call SID: {sid}") # Answer the call + self.call_mode = call_type self.set_avatar(peer_jid) self.status = "connecting" self.switch_mode("call") @@ -281,11 +418,18 @@ sdp: Session Description Protocol data @param profile: Profile associated """ - if self.sid != session_id: - log.debug( - f"Call ignored due to different session ID ({self.sid=} {session_id=})" - ) - return + if self.sid == session_id: + webrtc = self.webrtc + else: + for file_webrtc in self.files_webrtc: + webrtc = file_webrtc["webrtc"] + if webrtc.sid == session_id: + break + else: + log.debug( + f"Call ignored due to different session ID ({self.sid=} {session_id=})" + ) + return try: role = setup_data["role"] sdp = setup_data["sdp"] @@ -295,9 +439,9 @@ ) return if role == "initiator": - await self.webrtc.accept_call(session_id, sdp, profile) + await webrtc.accept_call(session_id, sdp, profile) elif role == "responder": - await self.webrtc.answer_call(session_id, sdp, profile) + await webrtc.answer_call(session_id, sdp, profile) else: dialog.notification.show( f"Invalid role received during setup: {setup_data}", level="error" @@ -358,20 +502,15 @@ # if there is any ringing, we stop it self.audio_player_elt.pause() self.audio_player_elt.currentTime = 0 + reason = data.get("reason", "") + text = data.get("text", "") if self.incoming_call_dialog_elt is not None: - self.incoming_call_dialog_elt.cancel() + self.incoming_call_dialog_elt.cancel(reason, text) self.incoming_call_dialog_elt = None self.switch_mode("search") - if data.get("reason") == "busy": - assert self._callee is not None - peer_name = cache.identities[self._callee]["nicknames"][0] - dialog.notification.show( - f"{peer_name} can't answer your call", - level="info", - ) await self.webrtc.end_call() @@ -540,6 +679,24 @@ def on_switch_camera(self, __) -> None: aio.run(self.webrtc.switch_camera()) + def on_send_file(self, __) -> None: + document["send_file_input"].click() + + def _on_send_input_change(self, evt) -> None: + aio.run(self.on_send_input_change(evt)) + + async def on_send_input_change(self, evt) -> None: + assert self._callee is not None + files = evt.currentTarget.files + for file in files: + webrtc = WebRTC(file_only=True) + self.files_webrtc.append({ + "file": file, + "webrtc": webrtc + }) + await webrtc.send_file(self._callee, file) + + def _on_entity_click(self, item: dict) -> None: aio.run(self.on_entity_click(item))
--- a/libervia/web/pages/calls/_browser/webrtc.py Tue Mar 05 16:40:25 2024 +0100 +++ b/libervia/web/pages/calls/_browser/webrtc.py Sat Apr 06 13:06:17 2024 +0200 @@ -2,9 +2,8 @@ import re from bridge import AsyncBridge as Bridge -from browser import aio, console as log, document, timer, window +from browser import aio, console as log, document, window import dialog -import errors from javascript import JSObject import jid @@ -13,7 +12,85 @@ bridge = Bridge() +class FileSender: + CHUNK_SIZE = 64 * 1024 + + def __init__(self, session_id, file, data_channel): + self.session_id = session_id + self.file = file + self.data_channel = data_channel + data_channel.bind("open", self._on_open) + self.offset = 0 + + def _on_open(self, __): + log.info(f"Data channel open, starting to send {self.file.name}.") + self.send_file() + + def _on_reader_load(self, event): + self.data_channel.send(event.target.result) + self.offset += self.CHUNK_SIZE + self.send_file() + + def send_file(self): + if self.offset < self.file.size: + chunk = self.file.slice(self.offset, self.offset + self.CHUNK_SIZE) + reader = window.FileReader.new() + reader.onload = self._on_reader_load + reader.readAsArrayBuffer(chunk) + else: + log.info(f"file {self.file.name} sent.") + self.data_channel.close() + if self.session_id is not None: + aio.run(bridge.call_end(self.session_id, "")) + + +class FileReceiver: + + def __init__(self, session_id: str, data_channel, extra_data: dict) -> None: + """Initializes the file receiver with a data channel. + + @param data_channel: The RTCDataChannel through which file data is received. + """ + self.session_id = session_id + self.data_channel = data_channel + self.received_chunks = [] + self.file_data = extra_data.get("file_data", {}) + data_channel.bind("message", self._on_message) + data_channel.bind("close", self._on_close) + log.debug("File receiver created.") + + def _on_message(self, event) -> None: + """Handles incoming message events from the data channel. + + @param event: The event containing the data chunk. + """ + self.received_chunks.append(event.data) + + def _on_close(self, __) -> None: + """Handles the data channel's close event. + + Assembles the received chunks into a Blob and triggers a file download. + """ + # The file is complete, we assemble the chunks in a blob + blob = window.Blob.new(self.received_chunks) + url = window.URL.createObjectURL(blob) + + # and create the <a> element to download the file. + a = document.createElement("a") + a.href = url + a.download = self.file_data.get("name", "received_file") + document.body.appendChild(a) + a.click() + + # We now clean up. + document.body.removeChild(a) + window.URL.revokeObjectURL(url) + log.info("File received.") + aio.run(bridge.call_end(self.session_id, "")) + + class WebRTC: + def __init__( self, screen_sharing_cb=None, @@ -22,7 +99,21 @@ on_connection_lost_cb=None, on_video_devices=None, on_reset_cb=None, + file_only: bool = False, + extra_data: dict|None = None ): + """Initialise WebRTC instance. + + @param screen_sharing_cb: callable function for screen sharing event + @param on_connection_established_cb: callable function for connection established event + @param on_reconnect_cb: called when a reconnection is triggered. + @param on_connection_lost_cb: called when the connection is lost. + @param on_video_devices: called when new video devices are set. + @param on_reset_cb: called on instance reset. + @param file_only: indicates a file transfer only session. + @param extra_data: optional dictionary containing additional data. + Notably used for file transfer, where ``file_data`` key is used. + """ # reset self.on_reset_cb = on_reset_cb self.reset_instance() @@ -42,8 +133,16 @@ self.has_multiple_cameras = False self.current_camera = None - # Initially populate the video devices list - aio.run(self._populate_video_devices()) + self.file_only = file_only + if not file_only: + # Initially populate the video devices list + aio.run(self._populate_video_devices()) + + # video elements + self.local_video_elt = document["local_video"] + self.remote_video_elt = document["remote_video"] + else: + self.file_sender = None # muting self.is_audio_muted = None @@ -53,9 +152,10 @@ self._is_sharing_screen = False self.screen_sharing_cb = screen_sharing_cb - # video elements - self.local_video_elt = document["local_video"] - self.remote_video_elt = document["remote_video"] + # extra + if extra_data is None: + extra_data = {} + self.extra_data = extra_data @property def is_sharing_screen(self) -> bool: @@ -73,7 +173,6 @@ self._peer_connection = None self._media_types = None self._media_types_inv = None - self._callee = None self.ufrag = None self.pwd = None self.sid = None @@ -82,6 +181,7 @@ self.remote_candidates_buffer = { "audio": {"candidates": []}, "video": {"candidates": []}, + "application": {"candidates": []}, } self.local_candidates_buffer = {} self.media_candidates = {} @@ -318,7 +418,7 @@ async def _create_peer_connection( self, - ): + ) -> JSObject: """Creates peer connection""" if self._peer_connection is not None: raise Exception("create_peer_connection can't be called twice!") @@ -354,6 +454,7 @@ self._peer_connection = peer_connection window.pc = self._peer_connection + return peer_connection async def _get_user_media(self, audio: bool = True, video: bool = True) -> None: """ @@ -585,20 +686,21 @@ @param candidates: Dictionary containing new ICE candidates """ log.debug(f"new peer candidates received: {candidates}") - # FIXME: workaround for https://github.com/brython-dev/brython/issues/2227, the - # following test raise a JS exception + try: + # FIXME: javascript.NULL must be used here, once we move to Brython 3.12.3+ remoteDescription_is_none = self._peer_connection.remoteDescription is None except Exception as e: + # FIXME: should be fine in Brython 3.12.3+ log.debug("Workaround for Brython bug activated.") remoteDescription_is_none = True if ( self._peer_connection is None - # or self._peer_connection.remoteDescription is None + # or self._peer_connection.remoteDescription is NULL or remoteDescription_is_none ): - for media_type in ("audio", "video"): + for media_type in ("audio", "video", "application"): media_candidates = candidates.get(media_type) if media_candidates: buffer = self.remote_candidates_buffer[media_type] @@ -610,12 +712,13 @@ try: sdp_mline_index = self.get_sdp_mline_index(media_type) except Exception as e: - log.warning(e) + log.warning(f"Can't get sdp_mline_index: {e}") continue - ice_candidate = window.RTCIceCandidate.new( - {"candidate": candidate_sdp, "sdpMLineIndex": sdp_mline_index} - ) - await self._peer_connection.addIceCandidate(ice_candidate) + else: + ice_candidate = window.RTCIceCandidate.new( + {"candidate": candidate_sdp, "sdpMLineIndex": sdp_mline_index} + ) + await self._peer_connection.addIceCandidate(ice_candidate) def on_track(self, event): """New track has been received from peer @@ -635,6 +738,14 @@ log.debug(f"on_negotiation_needed {event=}") # TODO + def _on_data_channel(self, event) -> None: + """Handles the data channel event from the peer connection. + + @param event: The event associated with the opening of a data channel. + """ + data_channel = event.channel + self.file_receiver = FileReceiver(self.sid, data_channel, self.extra_data) + async def answer_call(self, sid: str, offer_sdp: str, profile: str): """We respond to the call""" log.debug("answering call") @@ -647,7 +758,10 @@ ) await self.on_ice_candidates_new(self.remote_candidates_buffer) self.remote_candidates_buffer.clear() - await self._get_user_media() + if self.file_only: + self._peer_connection.bind("datachannel", self._on_data_channel) + else: + await self._get_user_media() # Gather local ICE candidates local_ice_data = await self._gather_ice_candidates(False) @@ -655,23 +769,13 @@ await bridge.call_answer_sdp(sid, self._peer_connection.localDescription.sdp) - async def make_call( - self, callee_jid: jid.JID, audio: bool = True, video: bool = True - ) -> None: - """Start a WebRTC call - - @param audio: True if an audio flux is required - @param video: True if a video flux is required - """ - await self._create_peer_connection() - await self._get_user_media(audio, video) + async def _get_call_data(self) -> dict: + """Start a WebRTC call""" await self._gather_ice_candidates(True) - call_data = {"sdp": self._peer_connection.localDescription.sdp} - log.info(f"calling {callee_jid!r}") - self.sid = await bridge.call_start(str(callee_jid), json.dumps(call_data)) - log.debug(f"Call SID: {self.sid}") + return {"sdp": self._peer_connection.localDescription.sdp} + async def _send_buffered_local_candidates(self) -> None: if self.local_candidates_buffer: log.debug( f"sending buffered local ICE candidates: {self.local_candidates_buffer}" @@ -684,16 +788,68 @@ "pwd": self.pwd, "candidates": candidates } - aio.run( - bridge.ice_candidates_add( - self.sid, - json.dumps( - ice_data - ), - ) + await bridge.ice_candidates_add( + self.sid, + json.dumps( + ice_data + ), ) self.local_candidates_buffer.clear() + async def make_call( + self, callee_jid: jid.JID, audio: bool = True, video: bool = True + ) -> None: + """ + @param audio: True if an audio flux is required + @param video: True if a video flux is required + """ + await self._create_peer_connection() + await self._get_user_media(audio, video) + call_data = await self._get_call_data() + log.info(f"calling {callee_jid!r}") + self.sid = await bridge.call_start(str(callee_jid), json.dumps(call_data)) + log.debug(f"Call SID: {self.sid}") + await self._send_buffered_local_candidates() + + def _on_opened_data_channel(self, event): + log.info("Datachannel has been opened.") + + async def send_file(self, callee_jid: jid.JID, file: JSObject) -> None: + assert self.file_only + peer_connection = await self._create_peer_connection() + data_channel = peer_connection.createDataChannel("file") + call_data = await self._get_call_data() + log.info(f"sending file to {callee_jid!r}") + file_meta = { + "size": file.size + } + if file.type: + file_meta["media_type"] = file.type + + try: + file_data = json.loads(await bridge.file_jingle_send( + str(callee_jid), + "", + file.name, + "", + json.dumps({ + "webrtc": True, + "call_data": call_data, + **file_meta + }) + )) + except Exception as e: + dialog.notification.show( + f"Can't send file: {e}", level="error" + ) + return + + self.sid = file_data["session_id"] + + log.debug(f"File Transfer SID: {self.sid}") + await self._send_buffered_local_candidates() + self.file_sender = FileSender(self.sid, file, data_channel) + async def end_call(self) -> None: """Stop streaming and clean instance""" if self._peer_connection is None:
--- a/libervia/web/server/restricted_bridge.py Tue Mar 05 16:40:25 2024 +0100 +++ b/libervia/web/server/restricted_bridge.py Sat Apr 06 13:06:17 2024 +0200 @@ -16,11 +16,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from libervia.backend.core import exceptions +from libervia.backend.core.log import getLogger from libervia.backend.tools.common import data_format -from libervia.backend.core import exceptions + from libervia.web.server.constants import Const as C +log = getLogger(__name__) + + class RestrictedBridge: """bridge with limited access, which can be used in browser @@ -94,6 +99,32 @@ return await self.host.bridge_call( "external_disco_get", entity, profile) + async def file_jingle_send( + self, + peer_jid: str, + filepath: str, + name: str, + file_desc: str, + extra_s: str, + profile: str + ) -> str: + self.no_service_profile(profile) + if filepath: + # The file sending must be done P2P from the browser directly (the file is + # from end-user machine), and its data must be set in "extra". + # "filepath" must NOT be used in this case, as it would link a local file + # (i.e. from the backend machine), which is an obvious security issue. + log.warning( + f'"filepath" user by {profile!r} in file_jingle_send, this is not ' + "allowed, hack attempt?" + ) + raise exceptions.PermissionError( + "Using a filepath is not allowed." + ) + return await self.host.bridge_call( + "file_jingle_send", peer_jid, "", name, file_desc, extra_s, profile + ) + async def history_get( self, from_jid: str,