# HG changeset patch
# User Goffi <goffi@goffi.org>
# Date 1715428942 -7200
# Node ID 6feac4a25e60755a3a1f5f4a0ada322d91caa8af
# Parent  d07838fc9d99d6dbf88a7fabb11c6cb6b27ca2cf
browser: Remote Control implementation:

- Add `cbor-x` JS dependency.
- In "Call" page, a Remote Control session can now be started. This is done by clicking on
  a search item 3 dots menu. Libervia Web will act as a controlling device. The call box
  is then adapted, and mouse/wheel and keyboard events are sent to remote, touch events
  are converted to mouse one.
- Some Brython 3.12* related changes.

rel 436

diff -r d07838fc9d99 -r 6feac4a25e60 libervia/web/pages/_browser/browser_meta.json
--- a/libervia/web/pages/_browser/browser_meta.json	Sat May 11 13:57:49 2024 +0200
+++ b/libervia/web/pages/_browser/browser_meta.json	Sat May 11 14:02:22 2024 +0200
@@ -7,7 +7,8 @@
                 "moment": "^2.29.4",
                 "ogv": "^1.8.9",
                 "tippy.js": "^6.3",
-                "emoji-picker-element": "^1.18"
+                "emoji-picker-element": "^1.18",
+                "cbor-x": "^1.5.9"
             }
         },
         "brython_map": {
@@ -33,6 +34,10 @@
                 "import_type": "module",
                 "path": "emoji-picker-element/picker.js",
                 "export": []
+            },
+            "cbor-x": {
+                "path": "cbor-x/dist/index.min.js",
+                "export": ["CBOR"]
             }
         }
     }
diff -r d07838fc9d99 -r 6feac4a25e60 libervia/web/pages/calls/_browser/__init__.py
--- a/libervia/web/pages/calls/_browser/__init__.py	Sat May 11 13:57:49 2024 +0200
+++ b/libervia/web/pages/calls/_browser/__init__.py	Sat May 11 14:02:22 2024 +0200
@@ -4,7 +4,7 @@
 from browser import aio, console as log, document, window
 from cache import cache
 import dialog
-from javascript import JSObject
+from javascript import JSObject, NULL
 from jid import JID
 from jid_search import JidSearch
 import loading
@@ -27,7 +27,8 @@
 )
 AUDIO = "audio"
 VIDEO = "video"
-ALLOWED_CALL_MODES = {AUDIO, VIDEO}
+REMOTE = "remote-control"
+ALLOWED_CALL_MODES = {AUDIO, VIDEO, REMOTE}
 INACTIVE_CLASS = "inactive"
 MUTED_CLASS = "muted"
 SCREEN_OFF_CLASS = "screen-off"
@@ -107,6 +108,9 @@
                     ".click-to-audio": lambda evt, item: aio.run(
                         self.on_entity_action(evt, AUDIO, item)
                     ),
+                    ".click-to-remote-control": lambda evt, item: aio.run(
+                        self.on_entity_action(evt, REMOTE, item)
+                    ),
                 },
             },
         )
@@ -155,13 +159,28 @@
         if mode in ALLOWED_CALL_MODES:
             if self._call_mode == mode:
                 return
+            log.debug("Switching to {mode} call mode.")
             self._call_mode = mode
-            with_video = mode == VIDEO
-            for elt in self.call_box_elt.select(".is-video-only"):
-                if with_video:
+            selector = ".is-video-only, .is-not-remote"
+            for elt in self.call_box_elt.select(selector):
+                if mode == VIDEO:
+                    # In video, all elements are visible.
                     elt.classList.remove("is-hidden")
+                elif mode == AUDIO:
+                    # In audio, we hide video-only elements.
+                    if elt.classList.contains("is-video-only"):
+                        elt.classList.add("is-hidden")
+                    else:
+                        elt.classList.remove("is-hidden")
+                elif mode == REMOTE:
+                    # In remote, we show all video element, except if they are
+                    # `is-not-remote`
+                    if elt.classList.contains("is-not-remote"):
+                        elt.classList.add("is-hidden")
+                    else:
+                        elt.classList.remove("is-hidden")
                 else:
-                    elt.classList.add("is-hidden")
+                    raise Exception("This line should never be reached.")
         else:
             raise ValueError("Invalid call mode")
 
@@ -190,7 +209,10 @@
         @param profile: Profile associated with the action
         """
         action_data = json.loads(action_data_s)
-        if action_data.get("type") == "confirm" and action_data.get("subtype") == "file":
+        if (
+            action_data.get("type") in ("confirm", "not_in_roster_leak")
+            and action_data.get("subtype") == "file"
+        ):
             aio.run(self.on_file_preflight(action_data, action_id))
         elif action_data.get("type") == "file":
             aio.run(self.on_file_proposal(action_data, action_id))
@@ -286,7 +308,7 @@
             # TODO: Check if any other frontend is connected for this profile, and refuse
             # the file if none is.
             return
-        if action_data.get("file_accepted", False):
+        if action_data.get("pre_accepted", False):
             # File proposal has already been accepted in preflight.
             accepted = True
         else:
@@ -472,13 +494,24 @@
             btn_elt.classList.remove(INACTIVE_CLASS, MUTED_CLASS, "is-warning")
             btn_elt.classList.add("is-success")
 
-    async def make_call(self, audio: bool = True, video: bool = True) -> None:
+    async def make_call(
+        self,
+        audio: bool = True,
+        video: bool = True,
+        remote: bool = False
+    ) -> None:
         """Start a WebRTC call
 
         @param audio: True if an audio flux is required
         @param video: True if a video flux is required
+        @param remote: True if this is a Remote Control session.
         """
-        self.call_mode = VIDEO if video else AUDIO
+        if remote:
+            self.call_mode = REMOTE
+        elif video:
+            self.call_mode = VIDEO
+        else:
+            self.call_mode = AUDIO
         try:
             callee_jid = JID(self.search_elt.value.strip())
             if not callee_jid.is_valid:
@@ -495,7 +528,12 @@
         self.set_avatar(callee_jid)
 
         self.switch_mode("call")
-        await self.webrtc.make_call(callee_jid, audio, video)
+        if remote:
+            await self.webrtc.start_remote_control(
+                callee_jid, audio, video
+            )
+        else:
+            await self.webrtc.make_call(callee_jid, audio, video)
 
     async def end_call(self, data: dict) -> None:
         """Stop streaming and clean instance"""
@@ -612,18 +650,17 @@
         @param fullscreen: if set, determine the fullscreen state; otherwise,
             the fullscreen mode will be toggled.
         """
-        do_fullscreen = (
-            document.fullscreenElement is None if fullscreen is None else fullscreen
-        )
+        if fullscreen is None:
+            fullscreen = document.fullscreenElement is NULL
 
         try:
-            if do_fullscreen:
-                if document.fullscreenElement is None:
+            if fullscreen:
+                if document.fullscreenElement is NULL:
                     self.call_box_elt.requestFullscreen()
                     document["full_screen_btn"].classList.add("is-hidden")
                     document["exit_full_screen_btn"].classList.remove("is-hidden")
             else:
-                if document.fullscreenElement is not None:
+                if document.fullscreenElement is not NULL:
                     document.exitFullscreen()
                     document["full_screen_btn"].classList.remove("is-hidden")
                     document["exit_full_screen_btn"].classList.add("is-hidden")
@@ -711,11 +748,15 @@
         evt.stopPropagation()
         if action == "menu":
             evt.currentTarget.parent.classList.toggle("is-active")
-        elif action in (VIDEO, AUDIO):
+        elif action in (VIDEO, AUDIO, REMOTE):
             self.search_elt.value = item["entity"]
             # we want the dropdown to be inactive
             evt.currentTarget.closest(".dropdown").classList.remove("is-active")
-            await self.make_call(video=action == VIDEO)
+            if action == REMOTE:
+                await self.make_call(audio=False, video=True, remote=True)
+
+            else:
+                await self.make_call(video=action == VIDEO)
 
 
 CallUI()
diff -r d07838fc9d99 -r 6feac4a25e60 libervia/web/pages/calls/_browser/webrtc.py
--- a/libervia/web/pages/calls/_browser/webrtc.py	Sat May 11 13:57:49 2024 +0200
+++ b/libervia/web/pages/calls/_browser/webrtc.py	Sat May 11 14:02:22 2024 +0200
@@ -2,10 +2,11 @@
 import re
 
 from bridge import AsyncBridge as Bridge
-from browser import aio, console as log, document, window
+from browser import aio, console as log, document, window, DOMNode
 import dialog
-from javascript import JSObject
+from javascript import JSObject, NULL
 import jid
+from js_modules.cbor_x import CBOR
 
 log.warning = log.warn
 profile = window.profile or ""
@@ -89,6 +90,189 @@
         aio.run(bridge.call_end(self.session_id, ""))
 
 
+class RemoteControler:
+    """Send input events to controlled device"""
+
+    def __init__(
+        self,
+        session_id: str,
+        capture_elt: DOMNode,
+        data_channel: JSObject,
+        simulate_mouse: bool = True
+    ) -> None:
+        """Initialize a RemoteControler instance.
+
+        @param session_id: ID of the Jingle Session
+        @param capture_elt: element where the input events are captured.
+        @param data_channel: WebRTCDataChannel instance linking to controlled device.
+        @simulate_mouse: if True, touch event will be converted to mouse events.
+        """
+        self.session_id = session_id
+        self.capture_elt = capture_elt
+        self.capture_elt.bind("click", self._on_capture_elt_click)
+        self.data_channel = data_channel
+        data_channel.bind("open", self._on_open)
+        self.simulate_mouse = simulate_mouse
+        self.last_mouse_position = (0, 0)
+
+    def _on_capture_elt_click(self, __):
+        self.capture_elt.focus()
+
+    def _on_open(self, __):
+        log.info(f"Data channel open, starting to send inputs.")
+        self.start_capture()
+
+    def start_capture(self) -> None:
+        """Start capturing input events to send them to the controlled device."""
+        for event_name in [
+                "mousedown",
+                "mouseup",
+                "mousemove",
+                "keydown",
+                "keyup",
+                "touchstart",
+                "touchend",
+                "touchmove",
+                "wheel",
+        ]:
+            self.capture_elt.bind(event_name, self._send_event)
+        self.capture_elt.focus()
+
+    def get_stream_coordinates(self, client_x: float, client_y: float) -> tuple[float, float]:
+        """Calculate coordinates relative to the actual video stream.
+
+        This method calculates the coordinates relative to the video content inside the <video>
+        element, considering any scaling or letterboxing due to aspect ratio differences.
+
+        @param client_x: The clientX value from the event, relative to the viewport.
+        @param client_y: The clientY value from the event, relative to the viewport.
+        @return: The x and y coordinates relative to the actual video stream.
+        """
+        video_element = self.capture_elt
+        video_rect = video_element.getBoundingClientRect()
+
+        # Calculate offsets relative to the capture element
+        element_x = client_x - video_rect.left
+        element_y = client_y - video_rect.top
+
+        element_width, element_height = video_rect.width, video_rect.height
+        stream_width, stream_height = video_element.videoWidth, video_element.videoHeight
+
+        if not all((element_width, element_height, stream_width, stream_height)):
+            log.warning("Invalid dimensions for video or element, using offsets.")
+            return element_x, element_y
+
+        element_aspect_ratio = element_width / element_height
+        stream_aspect_ratio = stream_width / stream_height
+
+        # Calculate scale and offset based on aspect ratio comparison
+        if stream_aspect_ratio > element_aspect_ratio:
+            # Video is more "wide" than the element: letterboxes will be top and bottom
+            scale = element_width / stream_width
+            scaled_height = stream_height * scale
+            offset_x, offset_y = 0, (element_height - scaled_height) / 2
+        else:
+            # Video is more "tall" than the element: letterboxes will be on the sides
+            scale = element_height / stream_height
+            scaled_width = stream_width * scale
+            offset_x, offset_y = (element_width - scaled_width) / 2, 0
+
+        # Mouse coordinates relative to the video stream
+        x = (element_x - offset_x) / scale
+        y = (element_y - offset_y) / scale
+
+        # Ensure the coordinates are within the bounds of the video stream
+        x = max(0.0, min(x, stream_width))
+        y = max(0.0, min(y, stream_height))
+
+        return x, y
+
+    def _send_event(self, event: JSObject) -> None:
+        """Serialize and send the event to the controlled device through the data channel."""
+        event.preventDefault()
+        serialized_event = self._serialize_event(event)
+        # TODO: we should join events instead
+        self.data_channel.send(CBOR.encode(serialized_event))
+
+    def _serialize_event(self, event: JSObject) -> dict[str, object]:
+        """Serialize event data for transmission.
+
+        @param event: an input event.
+        @return: event data to send to peer.
+        """
+        if event.type.startswith("key"):
+            ret = {
+                "type": event.type,
+                "key": event.key,
+            }
+            if event.location:
+                ret["location"] = event.location
+            return ret
+        elif event.type.startswith("mouse"):
+            x, y = self.get_stream_coordinates(event.clientX, event.clientY)
+            return {
+                "type": event.type,
+                "buttons": event.buttons,
+                "x": x,
+                "y": y,
+            }
+        elif event.type.startswith("touch"):
+            touches =  [
+                {
+                    "identifier": touch.identifier,
+                    **dict(zip(["x", "y"], self.get_stream_coordinates(
+                        touch.clientX,
+                        touch.clientY
+                    ))),
+                }
+                for touch in event.touches
+            ]
+            if self.simulate_mouse:
+                # In simulate mouse mode, we send mouse events.
+                if touches:
+                    touch_data = touches[0]
+                    x, y = touch_data["x"], touch_data["y"]
+                    self.last_mouse_position = (x, y)
+                else:
+                    x, y = self.last_mouse_position
+
+                mouse_event: dict[str, object] = {
+                    "x": x,
+                    "y": y,
+                }
+                if event.type == "touchstart":
+                    mouse_event.update({
+                        "type": "mousedown",
+                        "buttons": 1,
+                    })
+                elif event.type == "touchend":
+                    mouse_event.update({
+                        "type": "mouseup",
+                        "buttons": 1,
+                    })
+                elif event.type == "touchmove":
+                    mouse_event.update({
+                        "type": "mousemove",
+                    })
+                return mouse_event
+            else:
+                # Normal mode, with send touch events.
+                return {
+                    "type": event.type,
+                    "touches": touches
+                }
+        elif event.type == "wheel":
+            return {
+                "type": event.type,
+                "deltaX": event.deltaX,
+                "deltaY": event.deltaY,
+                "deltaZ": event.deltaZ,
+                "deltaMode": event.deltaMode,
+            }
+        else:
+            raise Exception(f"Internal Error: unexpected event {event.type=}")
+
+
 class WebRTC:
 
     def __init__(
@@ -100,12 +284,13 @@
         on_video_devices=None,
         on_reset_cb=None,
         file_only: bool = False,
-        extra_data: dict|None = None
+        extra_data: dict | None = None,
     ):
         """Initialise WebRTC instance.
 
         @param screen_sharing_cb: callable function for screen sharing event
-        @param on_connection_established_cb: callable function for connection established event
+        @param on_connection_established_cb: callable function for connection established
+            event
         @param on_reconnect_cb: called when a reconnection is triggered.
         @param on_connection_lost_cb: called when the connection is lost.
         @param on_video_devices: called when new video devices are set.
@@ -376,7 +561,8 @@
                 self.on_connection_established_cb()
         elif state == "failed":
             log.error(
-                "ICE connection failed. Check network connectivity and ICE configurations."
+                "ICE connection failed. Check network connectivity and ICE"
+                " configurations."
             )
         elif state == "disconnected":
             log.warning("ICE connection was disconnected.")
@@ -431,9 +617,9 @@
             if server["type"] == "stun":
                 ice_server["urls"] = f"stun:{server['host']}:{server['port']}"
             elif server["type"] == "turn":
-                ice_server[
-                    "urls"
-                ] = f"turn:{server['host']}:{server['port']}?transport={server['transport']}"
+                ice_server["urls"] = (
+                    f"turn:{server['host']}:{server['port']}?transport={server['transport']}"
+                )
                 ice_server["username"] = server["username"]
                 ice_server["credential"] = server["password"]
             ice_servers.append(ice_server)
@@ -496,9 +682,9 @@
                         track.stop()
 
             media_constraints = {
-                "video": {"deviceId": self.current_camera}
-                if self.current_camera
-                else True
+                "video": (
+                    {"deviceId": self.current_camera} if self.current_camera else True
+                )
             }
 
             new_stream = await window.navigator.mediaDevices.getUserMedia(
@@ -540,8 +726,8 @@
             await video_sender.replaceTrack(new_video_tracks[0])
 
             if screen:
-                # For screen sharing, we track the end event to properly stop the sharing when
-                # the user clicks on the browser's stop sharing dialog.
+                # For screen sharing, we track the end event to properly stop the sharing
+                # when the user clicks on the browser's stop sharing dialog.
                 def on_track_ended(event):
                     aio.run(self.toggle_screen_sharing())
 
@@ -687,18 +873,17 @@
         """
         log.debug(f"new peer candidates received: {candidates}")
 
-        try:
-            # FIXME: javascript.NULL must be used here, once we move to Brython 3.12.3+
-            remoteDescription_is_none = self._peer_connection.remoteDescription is None
-        except Exception as e:
-            # FIXME: should be fine in Brython 3.12.3+
-            log.debug("Workaround for Brython bug activated.")
-            remoteDescription_is_none = True
+        # try:
+        #     # FIXME: javascript.NULL must be used here, once we move to Brython 3.12.3+
+        #     remoteDescription_is_none = self._peer_connection.remoteDescription is None
+        # except Exception as e:
+        #     # FIXME: should be fine in Brython 3.12.3+
+        #     log.debug("Workaround for Brython bug activated.")
+        #     remoteDescription_is_none = True
 
         if (
             self._peer_connection is None
-            # or self._peer_connection.remoteDescription is NULL
-            or remoteDescription_is_none
+            or self._peer_connection.remoteDescription is NULL
         ):
             for media_type in ("audio", "video", "application"):
                 media_candidates = candidates.get(media_type)
@@ -786,13 +971,11 @@
                 ice_data[media_type] = {
                     "ufrag": self.ufrag,
                     "pwd": self.pwd,
-                    "candidates": candidates
+                    "candidates": candidates,
                 }
             await bridge.ice_candidates_add(
                 self.sid,
-                json.dumps(
-                    ice_data
-                ),
+                json.dumps(ice_data),
             )
             self.local_candidates_buffer.clear()
 
@@ -811,6 +994,53 @@
         log.debug(f"Call SID: {self.sid}")
         await self._send_buffered_local_candidates()
 
+    async def start_remote_control(
+        self, callee_jid: jid.JID, audio: bool = True, video: bool = True
+    ) -> None:
+        """Starts a Remote Control session.
+
+        If both audio and video are False, no screenshare will be done, the input will be
+        sent without feedback.
+        @param audio: True if an audio flux is required
+        @param video: True if a video flux is required
+        """
+        if audio or not video:
+            raise NotImplementedError("Only video screenshare is supported for now.")
+        peer_connection = await self._create_peer_connection()
+        if video:
+            peer_connection.addTransceiver("video", {"direction": "recvonly"})
+        data_channel = peer_connection.createDataChannel("remote-control")
+
+        call_data = await self._get_call_data()
+
+        try:
+            remote_control_data = json.loads(
+                await bridge.remote_control_start(
+                    str(callee_jid),
+                    json.dumps(
+                        {
+                            "devices": {
+                                "keyboard": {},
+                                "mouse": {},
+                                "wheel": {}
+                            },
+                            "call_data": call_data,
+                        }
+                    ),
+                )
+            )
+        except Exception as e:
+            dialog.notification.show(f"Can't start remote control: {e}", level="error")
+            return
+
+        self.sid = remote_control_data["session_id"]
+
+        log.debug(f"Remote Control SID: {self.sid}")
+        await self._send_buffered_local_candidates()
+        self.remote_controller = RemoteControler(
+            self.sid, self.remote_video_elt, data_channel
+        )
+
     def _on_opened_data_channel(self, event):
         log.info("Datachannel has been opened.")
 
@@ -820,28 +1050,22 @@
         data_channel = peer_connection.createDataChannel("file")
         call_data = await self._get_call_data()
         log.info(f"sending file to {callee_jid!r}")
-        file_meta = {
-            "size": file.size
-        }
+        file_meta = {"size": file.size}
         if file.type:
             file_meta["media_type"] = file.type
 
         try:
-            file_data = json.loads(await bridge.file_jingle_send(
+            file_data = json.loads(
+                await bridge.file_jingle_send(
                     str(callee_jid),
                     "",
                     file.name,
                     "",
-                    json.dumps({
-                        "webrtc": True,
-                        "call_data": call_data,
-                        **file_meta
-                    })
-            ))
+                    json.dumps({"webrtc": True, "call_data": call_data, **file_meta}),
+                )
+            )
         except Exception as e:
-            dialog.notification.show(
-                f"Can't send file: {e}", level="error"
-            )
+            dialog.notification.show(f"Can't send file: {e}", level="error")
             return
 
         self.sid = file_data["session_id"]
diff -r d07838fc9d99 -r 6feac4a25e60 libervia/web/server/restricted_bridge.py
--- a/libervia/web/server/restricted_bridge.py	Sat May 11 13:57:49 2024 +0200
+++ b/libervia/web/server/restricted_bridge.py	Sat May 11 14:02:22 2024 +0200
@@ -318,3 +318,12 @@
         return await self.host.bridge_call(
             "jid_search", search_term, options_s, profile
         )
+
+    async def remote_control_start(
+        self, peer_jid_s: str, extra_s: str, profile: str
+    ) -> None:
+        self.no_service_profile(profile)
+        return await self.host.bridge_call(
+            "remote_control_start", peer_jid_s, extra_s, profile
+        )
+