Mercurial > libervia-backend
annotate libervia/cli/call_gui.py @ 4240:79c8a70e1813
backend, frontend: prepare remote control:
This is a series of changes necessary to prepare the implementation of remote control
feature:
- XEP-0166: add a `priority` attribute to `ApplicationData`: this is needed when several
applications are working in a same session, to know which one must be handled first.
Will be used to make Remote Control have precedence over Call content.
- XEP-0166: `_call_plugins` is now async and is not used with `DeferredList` anymore: the
benefit to have methods called in parallels is very low, and it cause a lot of trouble
as we can't predict order. Methods are now called sequentially so workflow can be
predicted.
- XEP-0167: fix `senders` XMPP attribute <=> SDP mapping
- XEP-0234: preflight acceptance key is now `pre-accepted` instead of `file-accepted`, so
the same key can be used with other jingle applications.
- XEP-0167, XEP-0343: move some method to XEP-0167
- XEP-0353: use new `priority` feature to call preflight methods of applications according
to it.
- frontend (webrtc): refactor the sources/sink handling with a more flexible mechanism
based on Pydantic models. It is now possible to have has many Data Channel as necessary,
to have them in addition to A/V streams, to specify manually GStreamer sources and
sinks, etc.
- frontend (webrtc): rework of the pipeline to reduce latency.
- frontend: new `portal_desktop` method. Screenshare portal handling has been moved there,
and RemoteDesktop portal has been added.
- frontend (webrtc): fix `extract_ufrag_pwd` method.
rel 436
author | Goffi <goffi@goffi.org> |
---|---|
date | Sat, 11 May 2024 13:52:41 +0200 |
parents | d01b8d002619 |
children | 0d7bb4df2343 |
rev | line source |
---|---|
4206 | 1 #!/usr/bin/env python3 |
2 | |
3 # Libervia CLI | |
4 # Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org) | |
5 | |
6 # This program is free software: you can redistribute it and/or modify | |
7 # it under the terms of the GNU Affero General Public License as published by | |
8 # the Free Software Foundation, either version 3 of the License, or | |
9 # (at your option) any later version. | |
10 | |
11 # This program is distributed in the hope that it will be useful, | |
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 # GNU Affero General Public License for more details. | |
15 | |
16 # You should have received a copy of the GNU Affero General Public License | |
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | |
19 import asyncio | |
20 from functools import partial | |
21 from pathlib import Path | |
22 from typing import Callable, cast | |
23 | |
24 from PyQt6.QtCore import QPoint, QSize, Qt | |
25 from PyQt6.QtGui import ( | |
26 QCloseEvent, | |
27 QColor, | |
28 QIcon, | |
29 QImage, | |
30 QPainter, | |
31 QPen, | |
32 QPixmap, | |
33 QResizeEvent, | |
34 QTransform, | |
35 ) | |
36 from PyQt6.QtWidgets import ( | |
37 QApplication, | |
38 QDialog, | |
39 QDialogButtonBox, | |
40 QHBoxLayout, | |
41 QLabel, | |
42 QListWidget, | |
43 QListWidgetItem, | |
44 QMainWindow, | |
45 QPushButton, | |
46 QSizePolicy, | |
47 QVBoxLayout, | |
48 QWidget, | |
49 ) | |
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
50 import gi |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
51 |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
52 from libervia.backend.core.i18n import _ |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
53 from libervia.frontends.tools import aio, display_servers, webrtc |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
54 gi.require_versions({ |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
55 "Gst": "1.0", |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
56 "GstWebRTC": "1.0" |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
57 }) |
4206 | 58 from gi.repository import Gst |
59 | |
60 | |
61 | |
62 ICON_SIZE = QSize(45, 45) | |
63 BUTTON_SIZE = QSize(50, 50) | |
64 running = False | |
65 | |
66 | |
4233
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
67 aio.install_glib_asyncio_iteration() |
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
68 |
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
69 |
4206 | 70 class ActivableButton(QPushButton): |
71 def __init__(self, text, parent=None): | |
72 super().__init__(parent) | |
73 self._activated = True | |
74 self._activated_colour = "#47c68e" | |
75 self._deactivated_colour = "#ffe089" | |
76 self._line_colour = "#ff0000" | |
77 self._update_background_color() | |
78 | |
79 @property | |
80 def activated_colour(self) -> str: | |
81 return self._activated_colour | |
82 | |
83 @activated_colour.setter | |
84 def activated_colour(self, new_colour: str) -> None: | |
85 if new_colour != self._activated_colour: | |
86 self._activated_colour = new_colour | |
87 self._update_background_color() | |
88 | |
89 @property | |
90 def deactivated_colour(self) -> str: | |
91 return self._deactivated_colour | |
92 | |
93 @deactivated_colour.setter | |
94 def deactivated_colour(self, new_colour: str) -> None: | |
95 if new_colour != self._deactivated_colour: | |
96 self._deactivated_colour = new_colour | |
97 self._update_background_color() | |
98 | |
99 @property | |
100 def line_colour(self) -> str: | |
101 return self._line_colour | |
102 | |
103 @line_colour.setter | |
104 def line_colour(self, new_colour: str) -> None: | |
105 if new_colour != self._line_colour: | |
106 self._line_colour = new_colour | |
107 self.update() | |
108 | |
109 def paintEvent(self, a0): | |
110 super().paintEvent(a0) | |
111 | |
112 if not self._activated: | |
113 painter = QPainter(self) | |
114 painter.setRenderHint(QPainter.RenderHint.Antialiasing) | |
115 | |
116 line_color = QColor(self._line_colour) | |
117 line_width = 4 | |
118 cap_style = Qt.PenCapStyle.RoundCap | |
119 | |
120 pen = QPen(line_color, line_width, Qt.PenStyle.SolidLine) | |
121 pen.setCapStyle(cap_style) | |
122 painter.setPen(pen) | |
123 | |
124 margin = 5 | |
125 start_point = QPoint(margin, self.height() - margin) | |
126 end_point = QPoint(self.width() - margin, margin) | |
127 painter.drawLine(start_point, end_point) | |
128 | |
129 def _update_background_color(self): | |
130 if self._activated: | |
131 self.setStyleSheet(f"background-color: {self._activated_colour};") | |
132 else: | |
133 self.setStyleSheet(f"background-color: {self._deactivated_colour};") | |
134 self.update() | |
135 | |
136 @property | |
137 def activated(self): | |
138 return self._activated | |
139 | |
140 @activated.setter | |
141 def activated(self, value): | |
142 if self._activated != value: | |
143 self._activated = value | |
144 self._update_background_color() | |
145 | |
146 | |
147 class X11DesktopScreenDialog(QDialog): | |
148 def __init__(self, windows_data, parent=None): | |
149 super().__init__(parent) | |
150 self.__a_result = asyncio.get_running_loop().create_future() | |
151 self.setWindowTitle("Please select a window to share:") | |
152 self.resize(400, 300) | |
153 self.list_widget = QListWidget(self) | |
154 for window_data in windows_data: | |
155 item = QListWidgetItem(window_data["title"]) | |
156 item.setData(Qt.ItemDataRole.UserRole, window_data) | |
157 self.list_widget.addItem(item) | |
158 | |
159 self.buttonBox = QDialogButtonBox( | |
160 QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel | |
161 ) | |
162 self.buttonBox.accepted.connect(self.on_accepted) | |
163 self.buttonBox.rejected.connect(self.on_rejected) | |
164 | |
165 layout = QVBoxLayout(self) | |
166 layout.addWidget(self.list_widget) | |
167 layout.addWidget(self.buttonBox) | |
168 | |
169 def get_selected_window(self) -> dict | None: | |
170 selectedItem = self.list_widget.currentItem() | |
171 if selectedItem: | |
172 return selectedItem.data(Qt.ItemDataRole.UserRole) | |
173 return None | |
174 | |
175 def on_accepted(self): | |
176 self.__a_result.set_result(self.get_selected_window()) | |
177 self.close() | |
178 | |
179 def on_rejected(self): | |
180 self.__a_result.set_result(None) | |
181 self.close() | |
182 | |
183 def closeEvent(self, a0): | |
184 super().closeEvent(a0) | |
185 if not self.__a_result.done(): | |
186 self.__a_result.set_result(None) | |
187 | |
188 async def a_show(self) -> dict | None: | |
189 self.open() | |
190 return await self.__a_result | |
191 | |
192 | |
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
193 class AVCallUI(QMainWindow): |
4206 | 194 def __init__(self, host, icons_path: Path): |
195 super().__init__() | |
196 self.host = host | |
197 self.webrtc_call = None | |
198 self.icons_path = icons_path | |
199 self.initUI() | |
200 | |
201 @staticmethod | |
202 async def run_qt_loop(app): | |
203 while running: | |
204 app.sendPostedEvents() | |
205 await asyncio.sleep(0.1) | |
206 | |
207 @classmethod | |
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
208 async def run(cls, parent, call_data): |
4206 | 209 """Run PyQt loop and show the app""" |
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
210 media_dir = Path(await parent.host.bridge.config_get("", "media_dir")) |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
211 icons_path = media_dir / "fonts/fontello/svg" |
4206 | 212 app = QApplication([]) |
213 av_call_gui = cls(parent.host, icons_path) | |
214 av_call_gui.show() | |
4233
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
215 webrtc_call = await webrtc.WebRTCCall.make_webrtc_call( |
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
216 parent.host.bridge, |
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
217 parent.profile, |
4206 | 218 call_data, |
4240
79c8a70e1813
backend, frontend: prepare remote control:
Goffi <goffi@goffi.org>
parents:
4233
diff
changeset
|
219 sinks_data=webrtc.SinksApp( |
4206 | 220 local_video_cb=partial(av_call_gui.on_new_sample, video_stream="local"), |
221 remote_video_cb=partial(av_call_gui.on_new_sample, video_stream="remote"), | |
222 ), | |
4233
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
223 # we want to be sure that call is ended if user presses `Ctrl + c` or anything |
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
224 # else stops the session. |
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
225 on_call_setup_cb=lambda sid, profile: parent.host.add_on_quit_callback( |
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
226 parent.host.bridge.call_end, sid, "", profile |
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
227 ), |
d01b8d002619
cli (call, file), frontends: implement webRTC data channel transfer:
Goffi <goffi@goffi.org>
parents:
4210
diff
changeset
|
228 on_call_ended_cb=lambda sid, profile: parent.host.a_quit(), |
4206 | 229 ) |
230 av_call_gui.webrtc_call = webrtc_call | |
231 | |
232 global running | |
233 running = True | |
234 await cls.run_qt_loop(app) | |
235 await parent.host.a_quit() | |
236 | |
237 def initUI(self): | |
238 self.setGeometry(100, 100, 800, 600) | |
239 self.setWindowTitle("Call") | |
240 | |
241 # Main layouts | |
242 self.background_widget = QWidget(self) | |
243 self.foreground_widget = QWidget(self) | |
244 self.setCentralWidget(self.background_widget) | |
245 back_layout = QVBoxLayout(self.background_widget) | |
246 front_layout = QVBoxLayout(self.foreground_widget) | |
247 | |
248 # Remote video | |
249 self.remote_video_widget = QLabel(self) | |
250 self.remote_video_widget.setSizePolicy( | |
251 QSizePolicy.Policy.Ignored, QSizePolicy.Policy.Ignored | |
252 ) | |
253 back_layout.addWidget(self.remote_video_widget) | |
254 | |
255 # Fullscreen button | |
256 fullscreen_layout = QHBoxLayout() | |
257 front_layout.addLayout(fullscreen_layout) | |
258 fullscreen_layout.addStretch() | |
259 self.fullscreen_btn = QPushButton("", self) | |
260 self.fullscreen_btn.setFixedSize(BUTTON_SIZE) | |
261 self.fullscreen_icon_normal = QIcon(str(self.icons_path / "resize-full.svg")) | |
262 self.fullscreen_icon_fullscreen = QIcon(str(self.icons_path / "resize-small.svg")) | |
263 self.fullscreen_btn.setIcon(self.fullscreen_icon_normal) | |
264 self.fullscreen_btn.setIconSize(ICON_SIZE) | |
265 self.fullscreen_btn.clicked.connect(self.toggle_fullscreen) | |
266 fullscreen_layout.addWidget(self.fullscreen_btn) | |
267 | |
268 # Control buttons | |
269 self.control_buttons_layout = QHBoxLayout() | |
270 self.control_buttons_layout.setSpacing(40) | |
271 self.toggle_video_btn = cast( | |
272 ActivableButton, self.add_control_button("videocam", self.toggle_video) | |
273 ) | |
274 self.toggle_audio_btn = cast( | |
275 ActivableButton, self.add_control_button("volume-up", self.toggle_audio) | |
276 ) | |
277 self.share_desktop_btn = cast( | |
278 ActivableButton, self.add_control_button("desktop", self.share_desktop) | |
279 ) | |
280 self.share_desktop_btn.deactivated_colour = "#47c68e" | |
281 self.share_desktop_btn.activated_colour = "#f24468" | |
282 self.share_desktop_btn.line_colour = "#666666" | |
283 self.share_desktop_btn.activated = False | |
284 self.hang_up_btn = self.add_control_button( | |
285 "phone", self.hang_up, rotate=135, background="red", activable=False | |
286 ) | |
287 | |
288 controls_widget = QWidget(self) | |
289 controls_widget.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) | |
290 controls_widget.setLayout(self.control_buttons_layout) | |
291 front_layout.addStretch() | |
292 | |
293 bottom_layout = QHBoxLayout() | |
294 bottom_layout.addStretch() | |
295 front_layout.addLayout(bottom_layout) | |
296 bottom_layout.addWidget(controls_widget, alignment=Qt.AlignmentFlag.AlignBottom) | |
297 | |
298 # Local video feedback | |
299 bottom_layout.addStretch() | |
300 self.local_video_widget = QLabel(self) | |
301 bottom_layout.addWidget(self.local_video_widget) | |
302 | |
303 # we update sizes on resize event | |
304 self.background_widget.resizeEvent = self.adjust_sizes | |
305 self.adjust_sizes() | |
306 | |
307 def add_control_button( | |
308 self, | |
309 icon_name: str, | |
310 callback: Callable, | |
311 rotate: float | None = None, | |
312 background: str | None = None, | |
313 activable: bool = True, | |
314 ) -> QPushButton | ActivableButton: | |
315 if activable: | |
316 button = ActivableButton("", self) | |
317 else: | |
318 button = QPushButton("", self) | |
319 icon_path = self.icons_path / f"{icon_name}.svg" | |
320 button.setIcon(QIcon(str(icon_path))) | |
321 button.setIconSize(ICON_SIZE) | |
322 button.setFixedSize(BUTTON_SIZE) | |
323 if rotate is not None: | |
324 pixmap = button.icon().pixmap(ICON_SIZE) | |
325 transform = QTransform() | |
326 transform.rotate(rotate) | |
327 rotated_pixmap = pixmap.transformed(transform) | |
328 button.setIcon(QIcon(rotated_pixmap)) | |
329 if background: | |
330 button.setStyleSheet(f"background-color: {background};") | |
331 button.clicked.connect(callback) | |
332 self.control_buttons_layout.addWidget(button) | |
333 return button | |
334 | |
335 def adjust_sizes(self, a0: QResizeEvent | None = None) -> None: | |
336 self.foreground_widget.setGeometry( | |
337 0, 0, self.background_widget.width(), self.background_widget.height() | |
338 ) | |
339 self.local_video_widget.setFixedSize(QSize(self.width() // 3, self.height() // 3)) | |
340 if a0 is not None: | |
341 super().resizeEvent(a0) | |
342 | |
343 def on_new_sample(self, video_sink, video_stream: str) -> bool: | |
344 sample = video_sink.emit("pull-sample") | |
345 if sample is None: | |
346 return False | |
347 | |
348 video_pad = video_sink.get_static_pad("sink") | |
349 assert video_pad is not None | |
350 s = video_pad.get_current_caps().get_structure(0) | |
351 stream_size = (s.get_value("width"), s.get_value("height")) | |
352 self.host.loop.loop.call_soon_threadsafe( | |
353 self.update_sample, sample, stream_size, video_stream | |
354 ) | |
355 | |
356 return False | |
357 | |
358 def update_sample(self, sample, stream_size, video_stream: str) -> None: | |
359 if sample is None: | |
360 return | |
361 | |
362 video_widget = ( | |
363 self.remote_video_widget | |
364 if video_stream == "remote" | |
365 else self.local_video_widget | |
366 ) | |
367 | |
368 buf = sample.get_buffer() | |
369 result, mapinfo = buf.map(Gst.MapFlags.READ) | |
370 if result: | |
371 buffer = mapinfo.data | |
372 width, height = stream_size | |
373 qimage = QImage(buffer, width, height, QImage.Format.Format_RGB888) | |
374 pixmap = QPixmap.fromImage(qimage).scaled( | |
375 QSize(video_widget.width(), video_widget.height()), | |
376 Qt.AspectRatioMode.KeepAspectRatio, | |
377 ) | |
378 video_widget.setPixmap(pixmap) | |
379 video_widget.setAlignment(Qt.AlignmentFlag.AlignCenter) | |
380 | |
381 buf.unmap(mapinfo) | |
382 | |
383 def toggle_fullscreen(self): | |
384 fullscreen = not self.isFullScreen() | |
385 if fullscreen: | |
386 self.fullscreen_btn.setIcon(self.fullscreen_icon_fullscreen) | |
387 self.showFullScreen() | |
388 else: | |
389 self.fullscreen_btn.setIcon(self.fullscreen_icon_normal) | |
390 self.showNormal() | |
391 | |
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
392 def closeEvent(self, a0: QCloseEvent|None) -> None: |
4206 | 393 super().closeEvent(a0) |
394 global running | |
395 running = False | |
396 | |
397 def toggle_video(self): | |
398 assert self.webrtc_call is not None | |
399 self.webrtc_call.webrtc.video_muted = not self.webrtc_call.webrtc.video_muted | |
400 self.toggle_video_btn.activated = not self.webrtc_call.webrtc.video_muted | |
401 | |
402 def toggle_audio(self): | |
403 assert self.webrtc_call is not None | |
404 self.webrtc_call.webrtc.audio_muted = not self.webrtc_call.webrtc.audio_muted | |
405 self.toggle_audio_btn.activated = not self.webrtc_call.webrtc.audio_muted | |
406 | |
407 def share_desktop(self): | |
408 assert self.webrtc_call is not None | |
409 if self.webrtc_call.webrtc.desktop_sharing: | |
410 self.webrtc_call.webrtc.desktop_sharing = False | |
411 self.share_desktop_btn.activated = False | |
412 elif display_servers.detect() == display_servers.X11: | |
413 aio.run_async(self.show_X11_screen_dialog()) | |
414 else: | |
415 self.webrtc_call.webrtc.desktop_sharing = True | |
416 | |
417 def hang_up(self): | |
418 self.close() | |
419 | |
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
420 @staticmethod |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
421 def can_run(): |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
422 # if a known display server is detected, we should be able to run |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
423 return display_servers.detect() is not None |
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
424 |
4206 | 425 async def show_X11_screen_dialog(self): |
426 assert self.webrtc_call is not None | |
427 windows_data = display_servers.x11_list_windows() | |
428 dialog = X11DesktopScreenDialog(windows_data, self) | |
429 selected = await dialog.a_show() | |
430 if selected is not None: | |
431 xid = selected["id"] | |
432 self.webrtc_call.webrtc.desktop_sharing_data = {"xid": xid} | |
433 self.webrtc_call.webrtc.desktop_sharing = True | |
434 self.share_desktop_btn.activated = True |