Mercurial > libervia-backend

comparison libervia/frontends/tools/webrtc_remote_control.py @ 4242:8acf46ed7f36

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
frontends: remote control implementation: This is the frontends common part of remote control implementation. It handle the creation of WebRTC session, and management of inputs. For now the reception use freedesktop.org Desktop portal, and works mostly with Wayland based Desktop Environments. rel 436
author Goffi <goffi@goffi.org>
date Sat, 11 May 2024 13:52:43 +0200
parents
children 0d7bb4df2343
comparison
equal deleted inserted replaced
4241:898db6daf0d0 4242:8acf46ed7f36
1 #!/usr/bin/env python3
2
3 # Libervia Remote-Control via WebRTC implementation.
4 # Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org)
5
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
15
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
19
20 import asyncio
21 from functools import partial
22 import logging
23 from typing import Any, Awaitable, Callable
24
25 import gi
26 from gi.overrides.Gst import GLib
27 from gi.repository import GstWebRTC
28
29 import cbor2
30 from libervia.backend.core.i18n import _
31 from libervia.backend.tools.common import data_format
32 from libervia.frontends.tools import aio, jid, webrtc
33 from libervia.frontends.tools.webrtc_models import (
34 CallData,
35 SourcesNone,
36 SourcesPipeline,
37 )
38
39 gi.require_versions({"Gst": "1.0", "GstWebRTC": "1.0"})
40 OnOpenCbType = Callable[["WebRTCRemoteController"], None|Awaitable[None]]
41 MOUSE_BTN_LEFT = 0x110
42 MOUSE_BTN_RIGHT = 0x111
43 MOUSE_BTN_MIDDLE = 0x112
44 MOUSE_BTN_FORWARD = 0x115
45 MOUSE_BTN_BACK = 0x116
46
47
48 log = logging.getLogger(__name__)
49
50
51 class WebRTCRemoteController:
52
53 def __init__(
54 self,
55 bridge,
56 profile: str,
57 on_call_start_cb: Callable[[dict], Any] | None = None,
58 end_call_cb: Callable[[], Any] | None = None,
59 ) -> None:
60 """Initializes the File Sender.
61
62 @param bridge: An async Bridge instance.
63 @param profile: The profile name to be used.
64 @param on_call_start_cb: A blocking or async callable that accepts a dict as its
65 only argument.
66 @param end_call_cb: A callable to be invoked at the end of a call.
67 """
68
69 self.bridge = bridge
70 self.profile = profile
71 self.on_call_start_cb = on_call_start_cb
72 self.end_call_cb = end_call_cb
73 self.loop = asyncio.get_event_loop()
74 self.data_channel: GstWebRTC.WebRTCDataChannel|None = None
75
76 def send_input(self, input_data: dict) -> None:
77 """Send an input data to controlled device
78
79 @param input_data: data of the input event.
80 """
81 assert self.data_channel is not None
82 self.data_channel.send_data(GLib.Bytes(cbor2.dumps(input_data)))
83
84 async def _on_webrtc_call_start(
85 self,
86 options: dict,
87 callee: str,
88 call_data: dict,
89 profile: str,
90 ) -> str:
91 rc_data = {
92 "webrtc": True,
93 "call_data": call_data,
94 }
95 rc_data.update(options)
96 remote_control_data_s = await self.bridge.remote_control_start(
97 str(callee),
98 data_format.serialise(
99 rc_data
100 ),
101 profile,
102 )
103 remote_control_data = data_format.deserialise(remote_control_data_s)
104
105 if self.on_call_start_cb is not None:
106 await aio.maybe_async(self.on_call_start_cb(remote_control_data))
107 return remote_control_data["session_id"]
108
109 def _on_dc_close(self, webrtc_call, data_channel: GstWebRTC.WebRTCDataChannel):
110 if webrtc_call is not None:
111 aio.run_from_thread(self._end_call_and_quit, webrtc_call, loop=self.loop)
112
113 async def _end_call_and_quit(self, webrtc_call):
114 await webrtc_call.webrtc.end_call()
115 if self.end_call_cb is not None:
116 await aio.maybe_async(self.end_call_cb())
117
118 def _on_dc_open(
119 self, on_open_cb: OnOpenCbType, data_channel: GstWebRTC.WebRTCDataChannel
120 ) -> None:
121 """Called when datachannel is open"""
122 self.data_channel = data_channel
123 aio.run_from_thread(self.on_dc_opened, on_open_cb, loop=self.loop)
124
125 async def on_dc_opened(self, on_open_cb: OnOpenCbType) -> None:
126 await aio.maybe_async(on_open_cb(self))
127
128 async def start(
129 self,
130 callee: jid.JID,
131 options: dict,
132 on_open_cb: OnOpenCbType
133 ) -> None:
134 """Start a remote control session with ``callee``
135
136 @param callee: The JID of the recipient to send the file to.
137 @param options: Options such as which devices to control.
138 @param on_open_cb: Method to call when the Data Channel is open.
139 The WebRTCRemoteController instance used as argument can then be used to send
140 input events to received.
141 """
142 call_data = CallData(callee=callee)
143 self.webrtc_call = await webrtc.WebRTCCall.make_webrtc_call(
144 self.bridge,
145 self.profile,
146 call_data,
147 sources_data=webrtc.SourcesDataChannel(
148 dc_open_cb=partial(self._on_dc_open, on_open_cb),
149 ),
150 call_start_cb=partial(self._on_webrtc_call_start, options),
151 )
152
153
154 class WebRTCRemoteControlReceiver:
155
156 def __init__(
157 self, bridge, profile: str, on_close_cb: Callable[[], Any] | None = None,
158 verbose: bool = False
159 ) -> None:
160 """Initializes the File Receiver.
161
162 @param bridge: An async Bridge instance.
163 @param profile: The profile name to be used.
164 @param on_close_cb: Called when the Data Channel is closed.
165 @param verbose: if True, print input events.
166 """
167 self.bridge = bridge
168 self.profile = profile
169 self.on_close_cb = on_close_cb
170 self.loop = asyncio.get_event_loop()
171 self.desktop_portal = None
172 self.remote_desktop_data: dict|None = None
173 self.stream_node_id: int|None = None
174 self.verbose = verbose
175
176 async def do_input(self, data: dict) -> None:
177 assert self.desktop_portal is not None
178 try:
179 type_ = data["type"]
180 if type_.startswith("mouse"):
181 try:
182 try:
183 x, y = data["x"], data["y"]
184 except KeyError:
185 dx, dy = data["movementX"], data["movementY"]
186 await self.desktop_portal.notify_pointer_motion(
187 dx, dy
188 )
189 else:
190 assert self.stream_node_id is not None
191 await self.desktop_portal.notify_pointer_motion_absolute(
192 self.stream_node_id, x, y
193 )
194 except Exception:
195 log.exception("Can't send input")
196
197 if type_ in ("mouseup", "mousedown"):
198 buttons = data["buttons"]
199 state = 1 if type_ == "mousedown" else 0
200 # see https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/buttons#value
201 if buttons & 1:
202 await self.desktop_portal.notify_pointer_button(
203 MOUSE_BTN_LEFT,
204 state
205 )
206 if buttons & 2:
207 await self.desktop_portal.notify_pointer_button(
208 MOUSE_BTN_RIGHT,
209 state
210 )
211 if buttons & 4:
212 await self.desktop_portal.notify_pointer_button(
213 MOUSE_BTN_MIDDLE,
214 state
215 )
216 if buttons & 8:
217 await self.desktop_portal.notify_pointer_button(
218 MOUSE_BTN_BACK,
219 state
220 )
221 if buttons & 16:
222 await self.desktop_portal.notify_pointer_button(
223 MOUSE_BTN_FORWARD,
224 state
225 )
226 elif type_ == "wheel":
227 dx = data.get("deltaX", 0)
228 dy = data.get("deltaY", 0)
229 delta_mode = data["deltaMode"]
230 if delta_mode == 0:
231 # deltas are in pixels
232 await self.desktop_portal.notify_pointer_axis(
233 dx,
234 dy
235 )
236 else:
237 # deltas are in steps (see
238 # https://developer.mozilla.org/en-US/docs/Web/API/Element/wheel_event#event_properties)
239 if dx:
240 await self.desktop_portal.notify_pointer_axis(
241 1,
242 dx
243 )
244 if dy:
245 await self.desktop_portal.notify_pointer_axis(
246 0,
247 dy
248 )
249 elif type_.startswith("key"):
250 # FIXME: this is a really naive implementation, it needs tot be improved.
251 key = data["key"]
252 if data.get("shiftKey", False):
253 key = key.upper()
254 await self.desktop_portal.notify_keyboard_keysym(
255 ord(key), 1 if type_ == "keydown" else 0
256 )
257
258 except Exception:
259 log.exception(f"Can't handle input {data}")
260
261
262 def _on_dc_message_data(self, data_channel, glib_data) -> None:
263 """A data chunk of the file has been received."""
264 raw = glib_data.get_data()
265 data = cbor2.loads(raw)
266 if self.verbose:
267 print(data)
268 aio.run_from_thread(
269 self.do_input,
270 data,
271 loop=self.loop
272 )
273
274 def _on_dc_close(self, data_channel) -> None:
275 """Data channel is closed
276
277 The file download should be complete, we close it.
278 """
279 aio.run_from_thread(self._on_close, loop=self.loop)
280
281 async def _on_close(self) -> None:
282 if self.on_close_cb is not None:
283 await aio.maybe_async(self.on_close_cb())
284
285 def _on_data_channel(self, webrtcbin, data_channel) -> None:
286 """The data channel has been opened."""
287 data_channel.connect(
288 "on-message-data", self._on_dc_message_data
289 )
290 data_channel.connect("on-close", self._on_dc_close)
291
292 async def request_remote_desktop(self, with_screen_sharing: bool) -> None:
293 """Request autorisation to remote control desktop.
294
295 @param with_screen_sharing: True if screen must be shared.
296 """
297 from .portal_desktop import DesktopPortal
298 self.desktop_portal = DesktopPortal()
299 self.remote_desktop_data = await self.desktop_portal.request_remote_desktop(
300 with_screen_sharing
301 )
302 print(self.remote_desktop_data)
303
304 async def start_receiving(
305 self,
306 from_jid: jid.JID,
307 session_id: str,
308 screenshare: dict
309 ) -> None:
310 """Receives a file via WebRTC and saves it to the specified path.
311
312 @param from_jid: The JID of the entity sending the file.
313 @param session_id: The Jingle FT Session ID.
314 @param file_path: The local path where the received file will be saved.
315 If a file already exists at this path, it will be overwritten.
316 @param file_data: Additional data about the file being transferred.
317 """
318 call_data = CallData(callee=from_jid, sid=session_id)
319 if "video" in screenshare and self.remote_desktop_data:
320 try:
321 stream_data = self.remote_desktop_data["stream_data"]
322 log.debug(f"{stream_data=}")
323 self.stream_node_id = stream_data["node_id"]
324
325 sources_data = SourcesPipeline(
326 video_pipeline="pipewiresrc",
327 audio_pipeline="",
328 video_properties={
329 "path": str(self.stream_node_id),
330 "do-timestamp": 1,
331 }
332 )
333 except KeyError:
334 sources_data = SourcesNone()
335 else:
336 sources_data = SourcesNone()
337
338 await webrtc.WebRTCCall.make_webrtc_call(
339 self.bridge,
340 self.profile,
341 call_data,
342 sources_data = sources_data,
343 sinks_data=webrtc.SinksNone(),
344 dc_data_list=[webrtc.SinksDataChannel(
345 dc_on_data_channel=self._on_data_channel,
346 )],
347 )