Mercurial > libervia-web
comparison libervia/web/pages/calls/_browser/webrtc.py @ 1602:6feac4a25e60
browser: Remote Control implementation:
- Add `cbor-x` JS dependency.
- In "Call" page, a Remote Control session can now be started. This is done by clicking on
a search item 3 dots menu. Libervia Web will act as a controlling device. The call box
is then adapted, and mouse/wheel and keyboard events are sent to remote, touch events
are converted to mouse one.
- Some Brython 3.12* related changes.
rel 436
author | Goffi <goffi@goffi.org> |
---|---|
date | Sat, 11 May 2024 14:02:22 +0200 |
parents | 0a4433a343a3 |
children | 4a9679369856 |
comparison
equal
deleted
inserted
replaced
1601:d07838fc9d99 | 1602:6feac4a25e60 |
---|---|
1 import json | 1 import json |
2 import re | 2 import re |
3 | 3 |
4 from bridge import AsyncBridge as Bridge | 4 from bridge import AsyncBridge as Bridge |
5 from browser import aio, console as log, document, window | 5 from browser import aio, console as log, document, window, DOMNode |
6 import dialog | 6 import dialog |
7 from javascript import JSObject | 7 from javascript import JSObject, NULL |
8 import jid | 8 import jid |
9 from js_modules.cbor_x import CBOR | |
9 | 10 |
10 log.warning = log.warn | 11 log.warning = log.warn |
11 profile = window.profile or "" | 12 profile = window.profile or "" |
12 bridge = Bridge() | 13 bridge = Bridge() |
13 | 14 |
85 # We now clean up. | 86 # We now clean up. |
86 document.body.removeChild(a) | 87 document.body.removeChild(a) |
87 window.URL.revokeObjectURL(url) | 88 window.URL.revokeObjectURL(url) |
88 log.info("File received.") | 89 log.info("File received.") |
89 aio.run(bridge.call_end(self.session_id, "")) | 90 aio.run(bridge.call_end(self.session_id, "")) |
91 | |
92 | |
93 class RemoteControler: | |
94 """Send input events to controlled device""" | |
95 | |
96 def __init__( | |
97 self, | |
98 session_id: str, | |
99 capture_elt: DOMNode, | |
100 data_channel: JSObject, | |
101 simulate_mouse: bool = True | |
102 ) -> None: | |
103 """Initialize a RemoteControler instance. | |
104 | |
105 @param session_id: ID of the Jingle Session | |
106 @param capture_elt: element where the input events are captured. | |
107 @param data_channel: WebRTCDataChannel instance linking to controlled device. | |
108 @simulate_mouse: if True, touch event will be converted to mouse events. | |
109 """ | |
110 self.session_id = session_id | |
111 self.capture_elt = capture_elt | |
112 self.capture_elt.bind("click", self._on_capture_elt_click) | |
113 self.data_channel = data_channel | |
114 data_channel.bind("open", self._on_open) | |
115 self.simulate_mouse = simulate_mouse | |
116 self.last_mouse_position = (0, 0) | |
117 | |
118 def _on_capture_elt_click(self, __): | |
119 self.capture_elt.focus() | |
120 | |
121 def _on_open(self, __): | |
122 log.info(f"Data channel open, starting to send inputs.") | |
123 self.start_capture() | |
124 | |
125 def start_capture(self) -> None: | |
126 """Start capturing input events to send them to the controlled device.""" | |
127 for event_name in [ | |
128 "mousedown", | |
129 "mouseup", | |
130 "mousemove", | |
131 "keydown", | |
132 "keyup", | |
133 "touchstart", | |
134 "touchend", | |
135 "touchmove", | |
136 "wheel", | |
137 ]: | |
138 self.capture_elt.bind(event_name, self._send_event) | |
139 self.capture_elt.focus() | |
140 | |
141 def get_stream_coordinates(self, client_x: float, client_y: float) -> tuple[float, float]: | |
142 """Calculate coordinates relative to the actual video stream. | |
143 | |
144 This method calculates the coordinates relative to the video content inside the <video> | |
145 element, considering any scaling or letterboxing due to aspect ratio differences. | |
146 | |
147 @param client_x: The clientX value from the event, relative to the viewport. | |
148 @param client_y: The clientY value from the event, relative to the viewport. | |
149 @return: The x and y coordinates relative to the actual video stream. | |
150 """ | |
151 video_element = self.capture_elt | |
152 video_rect = video_element.getBoundingClientRect() | |
153 | |
154 # Calculate offsets relative to the capture element | |
155 element_x = client_x - video_rect.left | |
156 element_y = client_y - video_rect.top | |
157 | |
158 element_width, element_height = video_rect.width, video_rect.height | |
159 stream_width, stream_height = video_element.videoWidth, video_element.videoHeight | |
160 | |
161 if not all((element_width, element_height, stream_width, stream_height)): | |
162 log.warning("Invalid dimensions for video or element, using offsets.") | |
163 return element_x, element_y | |
164 | |
165 element_aspect_ratio = element_width / element_height | |
166 stream_aspect_ratio = stream_width / stream_height | |
167 | |
168 # Calculate scale and offset based on aspect ratio comparison | |
169 if stream_aspect_ratio > element_aspect_ratio: | |
170 # Video is more "wide" than the element: letterboxes will be top and bottom | |
171 scale = element_width / stream_width | |
172 scaled_height = stream_height * scale | |
173 offset_x, offset_y = 0, (element_height - scaled_height) / 2 | |
174 else: | |
175 # Video is more "tall" than the element: letterboxes will be on the sides | |
176 scale = element_height / stream_height | |
177 scaled_width = stream_width * scale | |
178 offset_x, offset_y = (element_width - scaled_width) / 2, 0 | |
179 | |
180 # Mouse coordinates relative to the video stream | |
181 x = (element_x - offset_x) / scale | |
182 y = (element_y - offset_y) / scale | |
183 | |
184 # Ensure the coordinates are within the bounds of the video stream | |
185 x = max(0.0, min(x, stream_width)) | |
186 y = max(0.0, min(y, stream_height)) | |
187 | |
188 return x, y | |
189 | |
190 def _send_event(self, event: JSObject) -> None: | |
191 """Serialize and send the event to the controlled device through the data channel.""" | |
192 event.preventDefault() | |
193 serialized_event = self._serialize_event(event) | |
194 # TODO: we should join events instead | |
195 self.data_channel.send(CBOR.encode(serialized_event)) | |
196 | |
197 def _serialize_event(self, event: JSObject) -> dict[str, object]: | |
198 """Serialize event data for transmission. | |
199 | |
200 @param event: an input event. | |
201 @return: event data to send to peer. | |
202 """ | |
203 if event.type.startswith("key"): | |
204 ret = { | |
205 "type": event.type, | |
206 "key": event.key, | |
207 } | |
208 if event.location: | |
209 ret["location"] = event.location | |
210 return ret | |
211 elif event.type.startswith("mouse"): | |
212 x, y = self.get_stream_coordinates(event.clientX, event.clientY) | |
213 return { | |
214 "type": event.type, | |
215 "buttons": event.buttons, | |
216 "x": x, | |
217 "y": y, | |
218 } | |
219 elif event.type.startswith("touch"): | |
220 touches = [ | |
221 { | |
222 "identifier": touch.identifier, | |
223 **dict(zip(["x", "y"], self.get_stream_coordinates( | |
224 touch.clientX, | |
225 touch.clientY | |
226 ))), | |
227 } | |
228 for touch in event.touches | |
229 ] | |
230 if self.simulate_mouse: | |
231 # In simulate mouse mode, we send mouse events. | |
232 if touches: | |
233 touch_data = touches[0] | |
234 x, y = touch_data["x"], touch_data["y"] | |
235 self.last_mouse_position = (x, y) | |
236 else: | |
237 x, y = self.last_mouse_position | |
238 | |
239 mouse_event: dict[str, object] = { | |
240 "x": x, | |
241 "y": y, | |
242 } | |
243 if event.type == "touchstart": | |
244 mouse_event.update({ | |
245 "type": "mousedown", | |
246 "buttons": 1, | |
247 }) | |
248 elif event.type == "touchend": | |
249 mouse_event.update({ | |
250 "type": "mouseup", | |
251 "buttons": 1, | |
252 }) | |
253 elif event.type == "touchmove": | |
254 mouse_event.update({ | |
255 "type": "mousemove", | |
256 }) | |
257 return mouse_event | |
258 else: | |
259 # Normal mode, with send touch events. | |
260 return { | |
261 "type": event.type, | |
262 "touches": touches | |
263 } | |
264 elif event.type == "wheel": | |
265 return { | |
266 "type": event.type, | |
267 "deltaX": event.deltaX, | |
268 "deltaY": event.deltaY, | |
269 "deltaZ": event.deltaZ, | |
270 "deltaMode": event.deltaMode, | |
271 } | |
272 else: | |
273 raise Exception(f"Internal Error: unexpected event {event.type=}") | |
90 | 274 |
91 | 275 |
92 class WebRTC: | 276 class WebRTC: |
93 | 277 |
94 def __init__( | 278 def __init__( |
98 on_reconnect_cb=None, | 282 on_reconnect_cb=None, |
99 on_connection_lost_cb=None, | 283 on_connection_lost_cb=None, |
100 on_video_devices=None, | 284 on_video_devices=None, |
101 on_reset_cb=None, | 285 on_reset_cb=None, |
102 file_only: bool = False, | 286 file_only: bool = False, |
103 extra_data: dict|None = None | 287 extra_data: dict | None = None, |
104 ): | 288 ): |
105 """Initialise WebRTC instance. | 289 """Initialise WebRTC instance. |
106 | 290 |
107 @param screen_sharing_cb: callable function for screen sharing event | 291 @param screen_sharing_cb: callable function for screen sharing event |
108 @param on_connection_established_cb: callable function for connection established event | 292 @param on_connection_established_cb: callable function for connection established |
293 event | |
109 @param on_reconnect_cb: called when a reconnection is triggered. | 294 @param on_reconnect_cb: called when a reconnection is triggered. |
110 @param on_connection_lost_cb: called when the connection is lost. | 295 @param on_connection_lost_cb: called when the connection is lost. |
111 @param on_video_devices: called when new video devices are set. | 296 @param on_video_devices: called when new video devices are set. |
112 @param on_reset_cb: called on instance reset. | 297 @param on_reset_cb: called on instance reset. |
113 @param file_only: indicates a file transfer only session. | 298 @param file_only: indicates a file transfer only session. |
374 if state == "connected": | 559 if state == "connected": |
375 if self.on_connection_established_cb is not None: | 560 if self.on_connection_established_cb is not None: |
376 self.on_connection_established_cb() | 561 self.on_connection_established_cb() |
377 elif state == "failed": | 562 elif state == "failed": |
378 log.error( | 563 log.error( |
379 "ICE connection failed. Check network connectivity and ICE configurations." | 564 "ICE connection failed. Check network connectivity and ICE" |
565 " configurations." | |
380 ) | 566 ) |
381 elif state == "disconnected": | 567 elif state == "disconnected": |
382 log.warning("ICE connection was disconnected.") | 568 log.warning("ICE connection was disconnected.") |
383 if self.on_connection_lost_cb is not None: | 569 if self.on_connection_lost_cb is not None: |
384 self.on_connection_lost_cb() | 570 self.on_connection_lost_cb() |
429 for server in external_disco: | 615 for server in external_disco: |
430 ice_server = {} | 616 ice_server = {} |
431 if server["type"] == "stun": | 617 if server["type"] == "stun": |
432 ice_server["urls"] = f"stun:{server['host']}:{server['port']}" | 618 ice_server["urls"] = f"stun:{server['host']}:{server['port']}" |
433 elif server["type"] == "turn": | 619 elif server["type"] == "turn": |
434 ice_server[ | 620 ice_server["urls"] = ( |
435 "urls" | 621 f"turn:{server['host']}:{server['port']}?transport={server['transport']}" |
436 ] = f"turn:{server['host']}:{server['port']}?transport={server['transport']}" | 622 ) |
437 ice_server["username"] = server["username"] | 623 ice_server["username"] = server["username"] |
438 ice_server["credential"] = server["password"] | 624 ice_server["credential"] = server["password"] |
439 ice_servers.append(ice_server) | 625 ice_servers.append(ice_server) |
440 | 626 |
441 rtc_configuration = {"iceServers": ice_servers} | 627 rtc_configuration = {"iceServers": ice_servers} |
494 for track in self.local_video_elt.srcObject.getTracks(): | 680 for track in self.local_video_elt.srcObject.getTracks(): |
495 if track.kind == "video": | 681 if track.kind == "video": |
496 track.stop() | 682 track.stop() |
497 | 683 |
498 media_constraints = { | 684 media_constraints = { |
499 "video": {"deviceId": self.current_camera} | 685 "video": ( |
500 if self.current_camera | 686 {"deviceId": self.current_camera} if self.current_camera else True |
501 else True | 687 ) |
502 } | 688 } |
503 | 689 |
504 new_stream = await window.navigator.mediaDevices.getUserMedia( | 690 new_stream = await window.navigator.mediaDevices.getUserMedia( |
505 media_constraints | 691 media_constraints |
506 ) | 692 ) |
538 ) | 724 ) |
539 if video_sender: | 725 if video_sender: |
540 await video_sender.replaceTrack(new_video_tracks[0]) | 726 await video_sender.replaceTrack(new_video_tracks[0]) |
541 | 727 |
542 if screen: | 728 if screen: |
543 # For screen sharing, we track the end event to properly stop the sharing when | 729 # For screen sharing, we track the end event to properly stop the sharing |
544 # the user clicks on the browser's stop sharing dialog. | 730 # when the user clicks on the browser's stop sharing dialog. |
545 def on_track_ended(event): | 731 def on_track_ended(event): |
546 aio.run(self.toggle_screen_sharing()) | 732 aio.run(self.toggle_screen_sharing()) |
547 | 733 |
548 new_video_tracks[0].bind("ended", on_track_ended) | 734 new_video_tracks[0].bind("ended", on_track_ended) |
549 | 735 |
685 | 871 |
686 @param candidates: Dictionary containing new ICE candidates | 872 @param candidates: Dictionary containing new ICE candidates |
687 """ | 873 """ |
688 log.debug(f"new peer candidates received: {candidates}") | 874 log.debug(f"new peer candidates received: {candidates}") |
689 | 875 |
690 try: | 876 # try: |
691 # FIXME: javascript.NULL must be used here, once we move to Brython 3.12.3+ | 877 # # FIXME: javascript.NULL must be used here, once we move to Brython 3.12.3+ |
692 remoteDescription_is_none = self._peer_connection.remoteDescription is None | 878 # remoteDescription_is_none = self._peer_connection.remoteDescription is None |
693 except Exception as e: | 879 # except Exception as e: |
694 # FIXME: should be fine in Brython 3.12.3+ | 880 # # FIXME: should be fine in Brython 3.12.3+ |
695 log.debug("Workaround for Brython bug activated.") | 881 # log.debug("Workaround for Brython bug activated.") |
696 remoteDescription_is_none = True | 882 # remoteDescription_is_none = True |
697 | 883 |
698 if ( | 884 if ( |
699 self._peer_connection is None | 885 self._peer_connection is None |
700 # or self._peer_connection.remoteDescription is NULL | 886 or self._peer_connection.remoteDescription is NULL |
701 or remoteDescription_is_none | |
702 ): | 887 ): |
703 for media_type in ("audio", "video", "application"): | 888 for media_type in ("audio", "video", "application"): |
704 media_candidates = candidates.get(media_type) | 889 media_candidates = candidates.get(media_type) |
705 if media_candidates: | 890 if media_candidates: |
706 buffer = self.remote_candidates_buffer[media_type] | 891 buffer = self.remote_candidates_buffer[media_type] |
784 ice_data = {} | 969 ice_data = {} |
785 for media_type, candidates in self.local_candidates_buffer.items(): | 970 for media_type, candidates in self.local_candidates_buffer.items(): |
786 ice_data[media_type] = { | 971 ice_data[media_type] = { |
787 "ufrag": self.ufrag, | 972 "ufrag": self.ufrag, |
788 "pwd": self.pwd, | 973 "pwd": self.pwd, |
789 "candidates": candidates | 974 "candidates": candidates, |
790 } | 975 } |
791 await bridge.ice_candidates_add( | 976 await bridge.ice_candidates_add( |
792 self.sid, | 977 self.sid, |
793 json.dumps( | 978 json.dumps(ice_data), |
794 ice_data | |
795 ), | |
796 ) | 979 ) |
797 self.local_candidates_buffer.clear() | 980 self.local_candidates_buffer.clear() |
798 | 981 |
799 async def make_call( | 982 async def make_call( |
800 self, callee_jid: jid.JID, audio: bool = True, video: bool = True | 983 self, callee_jid: jid.JID, audio: bool = True, video: bool = True |
809 log.info(f"calling {callee_jid!r}") | 992 log.info(f"calling {callee_jid!r}") |
810 self.sid = await bridge.call_start(str(callee_jid), json.dumps(call_data)) | 993 self.sid = await bridge.call_start(str(callee_jid), json.dumps(call_data)) |
811 log.debug(f"Call SID: {self.sid}") | 994 log.debug(f"Call SID: {self.sid}") |
812 await self._send_buffered_local_candidates() | 995 await self._send_buffered_local_candidates() |
813 | 996 |
997 async def start_remote_control( | |
998 self, callee_jid: jid.JID, audio: bool = True, video: bool = True | |
999 ) -> None: | |
1000 """Starts a Remote Control session. | |
1001 | |
1002 If both audio and video are False, no screenshare will be done, the input will be | |
1003 sent without feedback. | |
1004 @param audio: True if an audio flux is required | |
1005 @param video: True if a video flux is required | |
1006 """ | |
1007 if audio or not video: | |
1008 raise NotImplementedError("Only video screenshare is supported for now.") | |
1009 peer_connection = await self._create_peer_connection() | |
1010 if video: | |
1011 peer_connection.addTransceiver("video", {"direction": "recvonly"}) | |
1012 data_channel = peer_connection.createDataChannel("remote-control") | |
1013 | |
1014 call_data = await self._get_call_data() | |
1015 | |
1016 try: | |
1017 remote_control_data = json.loads( | |
1018 await bridge.remote_control_start( | |
1019 str(callee_jid), | |
1020 json.dumps( | |
1021 { | |
1022 "devices": { | |
1023 "keyboard": {}, | |
1024 "mouse": {}, | |
1025 "wheel": {} | |
1026 }, | |
1027 "call_data": call_data, | |
1028 } | |
1029 ), | |
1030 ) | |
1031 ) | |
1032 except Exception as e: | |
1033 dialog.notification.show(f"Can't start remote control: {e}", level="error") | |
1034 return | |
1035 | |
1036 self.sid = remote_control_data["session_id"] | |
1037 | |
1038 log.debug(f"Remote Control SID: {self.sid}") | |
1039 await self._send_buffered_local_candidates() | |
1040 self.remote_controller = RemoteControler( | |
1041 self.sid, self.remote_video_elt, data_channel | |
1042 ) | |
1043 | |
814 def _on_opened_data_channel(self, event): | 1044 def _on_opened_data_channel(self, event): |
815 log.info("Datachannel has been opened.") | 1045 log.info("Datachannel has been opened.") |
816 | 1046 |
817 async def send_file(self, callee_jid: jid.JID, file: JSObject) -> None: | 1047 async def send_file(self, callee_jid: jid.JID, file: JSObject) -> None: |
818 assert self.file_only | 1048 assert self.file_only |
819 peer_connection = await self._create_peer_connection() | 1049 peer_connection = await self._create_peer_connection() |
820 data_channel = peer_connection.createDataChannel("file") | 1050 data_channel = peer_connection.createDataChannel("file") |
821 call_data = await self._get_call_data() | 1051 call_data = await self._get_call_data() |
822 log.info(f"sending file to {callee_jid!r}") | 1052 log.info(f"sending file to {callee_jid!r}") |
823 file_meta = { | 1053 file_meta = {"size": file.size} |
824 "size": file.size | |
825 } | |
826 if file.type: | 1054 if file.type: |
827 file_meta["media_type"] = file.type | 1055 file_meta["media_type"] = file.type |
828 | 1056 |
829 try: | 1057 try: |
830 file_data = json.loads(await bridge.file_jingle_send( | 1058 file_data = json.loads( |
1059 await bridge.file_jingle_send( | |
831 str(callee_jid), | 1060 str(callee_jid), |
832 "", | 1061 "", |
833 file.name, | 1062 file.name, |
834 "", | 1063 "", |
835 json.dumps({ | 1064 json.dumps({"webrtc": True, "call_data": call_data, **file_meta}), |
836 "webrtc": True, | 1065 ) |
837 "call_data": call_data, | 1066 ) |
838 **file_meta | |
839 }) | |
840 )) | |
841 except Exception as e: | 1067 except Exception as e: |
842 dialog.notification.show( | 1068 dialog.notification.show(f"Can't send file: {e}", level="error") |
843 f"Can't send file: {e}", level="error" | |
844 ) | |
845 return | 1069 return |
846 | 1070 |
847 self.sid = file_data["session_id"] | 1071 self.sid = file_data["session_id"] |
848 | 1072 |
849 log.debug(f"File Transfer SID: {self.sid}") | 1073 log.debug(f"File Transfer SID: {self.sid}") |