Mercurial > libervia-backend
annotate libervia/cli/call_gui.py @ 4231:e11b13418ba6
plugin XEP-0353, XEP-0234, jingle: WebRTC data channel signaling implementation:
Implement XEP-0343: Signaling WebRTC Data Channels in Jingle. The current version of the
XEP (0.3.1) has no implementation and contains some flaws. After discussing this on xsf@,
Daniel (from Conversations) mentioned that they had a sprint with Larma (from Dino) to
work on another version and provided me with this link:
https://gist.github.com/iNPUTmice/6c56f3e948cca517c5fb129016d99e74 . I have used it for my
implementation.
This implementation reuses work done on Jingle A/V call (notably XEP-0176 and XEP-0167
plugins), with adaptations. When used, XEP-0234 will not handle the file itself as it
normally does. This is because WebRTC has several implementations (browser for web
interface, GStreamer for others), and file/data must be handled directly by the frontend.
This is particularly important for web frontends, as the file is not sent from the backend
but from the end-user's browser device.
Among the changes, there are:
- XEP-0343 implementation.
- `file_send` bridge method now use serialised dict as output.
- New `BaseTransportHandler.is_usable` method which get content data and returns a boolean
(default to `True`) to tell if this transport can actually be used in this context (when
we are initiator). Used in webRTC case to see if call data are available.
- Support of `application` media type, and everything necessary to handle data channels.
- Better confirmation message, with file name, size and description when available.
- When file is accepted in preflight, it is specified in following `action_new` signal for
actual file transfer. This way, frontend can avoid the display or 2 confirmation
messages.
- XEP-0166: when not specified, default `content` name is now its index number instead of
a UUID. This follows the behaviour of browsers.
- XEP-0353: better handling of events such as call taken by another device.
- various other updates.
rel 441
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Sat, 06 Apr 2024 12:57:23 +0200 |
| parents | 9218d4331bb2 |
| children | d01b8d002619 |
| rev | line source |
|---|---|
| 4206 | 1 #!/usr/bin/env python3 |
| 2 | |
| 3 # Libervia CLI | |
| 4 # Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org) | |
| 5 | |
| 6 # This program is free software: you can redistribute it and/or modify | |
| 7 # it under the terms of the GNU Affero General Public License as published by | |
| 8 # the Free Software Foundation, either version 3 of the License, or | |
| 9 # (at your option) any later version. | |
| 10 | |
| 11 # This program is distributed in the hope that it will be useful, | |
| 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 14 # GNU Affero General Public License for more details. | |
| 15 | |
| 16 # You should have received a copy of the GNU Affero General Public License | |
| 17 # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 18 | |
| 19 import asyncio | |
| 20 from functools import partial | |
| 21 from pathlib import Path | |
| 22 from typing import Callable, cast | |
| 23 | |
| 24 from PyQt6.QtCore import QPoint, QSize, Qt | |
| 25 from PyQt6.QtGui import ( | |
| 26 QCloseEvent, | |
| 27 QColor, | |
| 28 QIcon, | |
| 29 QImage, | |
| 30 QPainter, | |
| 31 QPen, | |
| 32 QPixmap, | |
| 33 QResizeEvent, | |
| 34 QTransform, | |
| 35 ) | |
| 36 from PyQt6.QtWidgets import ( | |
| 37 QApplication, | |
| 38 QDialog, | |
| 39 QDialogButtonBox, | |
| 40 QHBoxLayout, | |
| 41 QLabel, | |
| 42 QListWidget, | |
| 43 QListWidgetItem, | |
| 44 QMainWindow, | |
| 45 QPushButton, | |
| 46 QSizePolicy, | |
| 47 QVBoxLayout, | |
| 48 QWidget, | |
| 49 ) | |
|
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
50 import gi |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
51 |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
52 from libervia.backend.core.i18n import _ |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
53 from libervia.cli.call_webrtc import WebRTCCall |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
54 from libervia.frontends.tools import aio, display_servers, webrtc |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
55 gi.require_versions({ |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
56 "Gst": "1.0", |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
57 "GstWebRTC": "1.0" |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
58 }) |
| 4206 | 59 from gi.repository import Gst |
| 60 | |
| 61 | |
| 62 | |
| 63 ICON_SIZE = QSize(45, 45) | |
| 64 BUTTON_SIZE = QSize(50, 50) | |
| 65 running = False | |
| 66 | |
| 67 | |
| 68 class ActivableButton(QPushButton): | |
| 69 def __init__(self, text, parent=None): | |
| 70 super().__init__(parent) | |
| 71 self._activated = True | |
| 72 self._activated_colour = "#47c68e" | |
| 73 self._deactivated_colour = "#ffe089" | |
| 74 self._line_colour = "#ff0000" | |
| 75 self._update_background_color() | |
| 76 | |
| 77 @property | |
| 78 def activated_colour(self) -> str: | |
| 79 return self._activated_colour | |
| 80 | |
| 81 @activated_colour.setter | |
| 82 def activated_colour(self, new_colour: str) -> None: | |
| 83 if new_colour != self._activated_colour: | |
| 84 self._activated_colour = new_colour | |
| 85 self._update_background_color() | |
| 86 | |
| 87 @property | |
| 88 def deactivated_colour(self) -> str: | |
| 89 return self._deactivated_colour | |
| 90 | |
| 91 @deactivated_colour.setter | |
| 92 def deactivated_colour(self, new_colour: str) -> None: | |
| 93 if new_colour != self._deactivated_colour: | |
| 94 self._deactivated_colour = new_colour | |
| 95 self._update_background_color() | |
| 96 | |
| 97 @property | |
| 98 def line_colour(self) -> str: | |
| 99 return self._line_colour | |
| 100 | |
| 101 @line_colour.setter | |
| 102 def line_colour(self, new_colour: str) -> None: | |
| 103 if new_colour != self._line_colour: | |
| 104 self._line_colour = new_colour | |
| 105 self.update() | |
| 106 | |
| 107 def paintEvent(self, a0): | |
| 108 super().paintEvent(a0) | |
| 109 | |
| 110 if not self._activated: | |
| 111 painter = QPainter(self) | |
| 112 painter.setRenderHint(QPainter.RenderHint.Antialiasing) | |
| 113 | |
| 114 line_color = QColor(self._line_colour) | |
| 115 line_width = 4 | |
| 116 cap_style = Qt.PenCapStyle.RoundCap | |
| 117 | |
| 118 pen = QPen(line_color, line_width, Qt.PenStyle.SolidLine) | |
| 119 pen.setCapStyle(cap_style) | |
| 120 painter.setPen(pen) | |
| 121 | |
| 122 margin = 5 | |
| 123 start_point = QPoint(margin, self.height() - margin) | |
| 124 end_point = QPoint(self.width() - margin, margin) | |
| 125 painter.drawLine(start_point, end_point) | |
| 126 | |
| 127 def _update_background_color(self): | |
| 128 if self._activated: | |
| 129 self.setStyleSheet(f"background-color: {self._activated_colour};") | |
| 130 else: | |
| 131 self.setStyleSheet(f"background-color: {self._deactivated_colour};") | |
| 132 self.update() | |
| 133 | |
| 134 @property | |
| 135 def activated(self): | |
| 136 return self._activated | |
| 137 | |
| 138 @activated.setter | |
| 139 def activated(self, value): | |
| 140 if self._activated != value: | |
| 141 self._activated = value | |
| 142 self._update_background_color() | |
| 143 | |
| 144 | |
| 145 class X11DesktopScreenDialog(QDialog): | |
| 146 def __init__(self, windows_data, parent=None): | |
| 147 super().__init__(parent) | |
| 148 self.__a_result = asyncio.get_running_loop().create_future() | |
| 149 self.setWindowTitle("Please select a window to share:") | |
| 150 self.resize(400, 300) | |
| 151 self.list_widget = QListWidget(self) | |
| 152 for window_data in windows_data: | |
| 153 item = QListWidgetItem(window_data["title"]) | |
| 154 item.setData(Qt.ItemDataRole.UserRole, window_data) | |
| 155 self.list_widget.addItem(item) | |
| 156 | |
| 157 self.buttonBox = QDialogButtonBox( | |
| 158 QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel | |
| 159 ) | |
| 160 self.buttonBox.accepted.connect(self.on_accepted) | |
| 161 self.buttonBox.rejected.connect(self.on_rejected) | |
| 162 | |
| 163 layout = QVBoxLayout(self) | |
| 164 layout.addWidget(self.list_widget) | |
| 165 layout.addWidget(self.buttonBox) | |
| 166 | |
| 167 def get_selected_window(self) -> dict | None: | |
| 168 selectedItem = self.list_widget.currentItem() | |
| 169 if selectedItem: | |
| 170 return selectedItem.data(Qt.ItemDataRole.UserRole) | |
| 171 return None | |
| 172 | |
| 173 def on_accepted(self): | |
| 174 self.__a_result.set_result(self.get_selected_window()) | |
| 175 self.close() | |
| 176 | |
| 177 def on_rejected(self): | |
| 178 self.__a_result.set_result(None) | |
| 179 self.close() | |
| 180 | |
| 181 def closeEvent(self, a0): | |
| 182 super().closeEvent(a0) | |
| 183 if not self.__a_result.done(): | |
| 184 self.__a_result.set_result(None) | |
| 185 | |
| 186 async def a_show(self) -> dict | None: | |
| 187 self.open() | |
| 188 return await self.__a_result | |
| 189 | |
| 190 | |
|
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
191 class AVCallUI(QMainWindow): |
| 4206 | 192 def __init__(self, host, icons_path: Path): |
| 193 super().__init__() | |
| 194 self.host = host | |
| 195 self.webrtc_call = None | |
| 196 self.icons_path = icons_path | |
| 197 self.initUI() | |
| 198 | |
| 199 @staticmethod | |
| 200 async def run_qt_loop(app): | |
| 201 while running: | |
| 202 app.sendPostedEvents() | |
| 203 await asyncio.sleep(0.1) | |
| 204 | |
| 205 @classmethod | |
|
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
206 async def run(cls, parent, call_data): |
| 4206 | 207 """Run PyQt loop and show the app""" |
|
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
208 media_dir = Path(await parent.host.bridge.config_get("", "media_dir")) |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
209 icons_path = media_dir / "fonts/fontello/svg" |
| 4206 | 210 app = QApplication([]) |
| 211 av_call_gui = cls(parent.host, icons_path) | |
| 212 av_call_gui.show() | |
|
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
213 webrtc_call = await WebRTCCall.make_webrtc_call( |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
214 parent.host, |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
215 parent.profile, |
| 4206 | 216 call_data, |
| 217 sinks=webrtc.SINKS_APP, | |
| 218 appsink_data=webrtc.AppSinkData( | |
| 219 local_video_cb=partial(av_call_gui.on_new_sample, video_stream="local"), | |
| 220 remote_video_cb=partial(av_call_gui.on_new_sample, video_stream="remote"), | |
| 221 ), | |
| 222 ) | |
| 223 av_call_gui.webrtc_call = webrtc_call | |
| 224 | |
| 225 global running | |
| 226 running = True | |
| 227 await cls.run_qt_loop(app) | |
| 228 await parent.host.a_quit() | |
| 229 | |
| 230 def initUI(self): | |
| 231 self.setGeometry(100, 100, 800, 600) | |
| 232 self.setWindowTitle("Call") | |
| 233 | |
| 234 # Main layouts | |
| 235 self.background_widget = QWidget(self) | |
| 236 self.foreground_widget = QWidget(self) | |
| 237 self.setCentralWidget(self.background_widget) | |
| 238 back_layout = QVBoxLayout(self.background_widget) | |
| 239 front_layout = QVBoxLayout(self.foreground_widget) | |
| 240 | |
| 241 # Remote video | |
| 242 self.remote_video_widget = QLabel(self) | |
| 243 self.remote_video_widget.setSizePolicy( | |
| 244 QSizePolicy.Policy.Ignored, QSizePolicy.Policy.Ignored | |
| 245 ) | |
| 246 back_layout.addWidget(self.remote_video_widget) | |
| 247 | |
| 248 # Fullscreen button | |
| 249 fullscreen_layout = QHBoxLayout() | |
| 250 front_layout.addLayout(fullscreen_layout) | |
| 251 fullscreen_layout.addStretch() | |
| 252 self.fullscreen_btn = QPushButton("", self) | |
| 253 self.fullscreen_btn.setFixedSize(BUTTON_SIZE) | |
| 254 self.fullscreen_icon_normal = QIcon(str(self.icons_path / "resize-full.svg")) | |
| 255 self.fullscreen_icon_fullscreen = QIcon(str(self.icons_path / "resize-small.svg")) | |
| 256 self.fullscreen_btn.setIcon(self.fullscreen_icon_normal) | |
| 257 self.fullscreen_btn.setIconSize(ICON_SIZE) | |
| 258 self.fullscreen_btn.clicked.connect(self.toggle_fullscreen) | |
| 259 fullscreen_layout.addWidget(self.fullscreen_btn) | |
| 260 | |
| 261 # Control buttons | |
| 262 self.control_buttons_layout = QHBoxLayout() | |
| 263 self.control_buttons_layout.setSpacing(40) | |
| 264 self.toggle_video_btn = cast( | |
| 265 ActivableButton, self.add_control_button("videocam", self.toggle_video) | |
| 266 ) | |
| 267 self.toggle_audio_btn = cast( | |
| 268 ActivableButton, self.add_control_button("volume-up", self.toggle_audio) | |
| 269 ) | |
| 270 self.share_desktop_btn = cast( | |
| 271 ActivableButton, self.add_control_button("desktop", self.share_desktop) | |
| 272 ) | |
| 273 self.share_desktop_btn.deactivated_colour = "#47c68e" | |
| 274 self.share_desktop_btn.activated_colour = "#f24468" | |
| 275 self.share_desktop_btn.line_colour = "#666666" | |
| 276 self.share_desktop_btn.activated = False | |
| 277 self.hang_up_btn = self.add_control_button( | |
| 278 "phone", self.hang_up, rotate=135, background="red", activable=False | |
| 279 ) | |
| 280 | |
| 281 controls_widget = QWidget(self) | |
| 282 controls_widget.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) | |
| 283 controls_widget.setLayout(self.control_buttons_layout) | |
| 284 front_layout.addStretch() | |
| 285 | |
| 286 bottom_layout = QHBoxLayout() | |
| 287 bottom_layout.addStretch() | |
| 288 front_layout.addLayout(bottom_layout) | |
| 289 bottom_layout.addWidget(controls_widget, alignment=Qt.AlignmentFlag.AlignBottom) | |
| 290 | |
| 291 # Local video feedback | |
| 292 bottom_layout.addStretch() | |
| 293 self.local_video_widget = QLabel(self) | |
| 294 bottom_layout.addWidget(self.local_video_widget) | |
| 295 | |
| 296 # we update sizes on resize event | |
| 297 self.background_widget.resizeEvent = self.adjust_sizes | |
| 298 self.adjust_sizes() | |
| 299 | |
| 300 def add_control_button( | |
| 301 self, | |
| 302 icon_name: str, | |
| 303 callback: Callable, | |
| 304 rotate: float | None = None, | |
| 305 background: str | None = None, | |
| 306 activable: bool = True, | |
| 307 ) -> QPushButton | ActivableButton: | |
| 308 if activable: | |
| 309 button = ActivableButton("", self) | |
| 310 else: | |
| 311 button = QPushButton("", self) | |
| 312 icon_path = self.icons_path / f"{icon_name}.svg" | |
| 313 button.setIcon(QIcon(str(icon_path))) | |
| 314 button.setIconSize(ICON_SIZE) | |
| 315 button.setFixedSize(BUTTON_SIZE) | |
| 316 if rotate is not None: | |
| 317 pixmap = button.icon().pixmap(ICON_SIZE) | |
| 318 transform = QTransform() | |
| 319 transform.rotate(rotate) | |
| 320 rotated_pixmap = pixmap.transformed(transform) | |
| 321 button.setIcon(QIcon(rotated_pixmap)) | |
| 322 if background: | |
| 323 button.setStyleSheet(f"background-color: {background};") | |
| 324 button.clicked.connect(callback) | |
| 325 self.control_buttons_layout.addWidget(button) | |
| 326 return button | |
| 327 | |
| 328 def adjust_sizes(self, a0: QResizeEvent | None = None) -> None: | |
| 329 self.foreground_widget.setGeometry( | |
| 330 0, 0, self.background_widget.width(), self.background_widget.height() | |
| 331 ) | |
| 332 self.local_video_widget.setFixedSize(QSize(self.width() // 3, self.height() // 3)) | |
| 333 if a0 is not None: | |
| 334 super().resizeEvent(a0) | |
| 335 | |
| 336 def on_new_sample(self, video_sink, video_stream: str) -> bool: | |
| 337 sample = video_sink.emit("pull-sample") | |
| 338 if sample is None: | |
| 339 return False | |
| 340 | |
| 341 video_pad = video_sink.get_static_pad("sink") | |
| 342 assert video_pad is not None | |
| 343 s = video_pad.get_current_caps().get_structure(0) | |
| 344 stream_size = (s.get_value("width"), s.get_value("height")) | |
| 345 self.host.loop.loop.call_soon_threadsafe( | |
| 346 self.update_sample, sample, stream_size, video_stream | |
| 347 ) | |
| 348 | |
| 349 return False | |
| 350 | |
| 351 def update_sample(self, sample, stream_size, video_stream: str) -> None: | |
| 352 if sample is None: | |
| 353 return | |
| 354 | |
| 355 video_widget = ( | |
| 356 self.remote_video_widget | |
| 357 if video_stream == "remote" | |
| 358 else self.local_video_widget | |
| 359 ) | |
| 360 | |
| 361 buf = sample.get_buffer() | |
| 362 result, mapinfo = buf.map(Gst.MapFlags.READ) | |
| 363 if result: | |
| 364 buffer = mapinfo.data | |
| 365 width, height = stream_size | |
| 366 qimage = QImage(buffer, width, height, QImage.Format.Format_RGB888) | |
| 367 pixmap = QPixmap.fromImage(qimage).scaled( | |
| 368 QSize(video_widget.width(), video_widget.height()), | |
| 369 Qt.AspectRatioMode.KeepAspectRatio, | |
| 370 ) | |
| 371 video_widget.setPixmap(pixmap) | |
| 372 video_widget.setAlignment(Qt.AlignmentFlag.AlignCenter) | |
| 373 | |
| 374 buf.unmap(mapinfo) | |
| 375 | |
| 376 def toggle_fullscreen(self): | |
| 377 fullscreen = not self.isFullScreen() | |
| 378 if fullscreen: | |
| 379 self.fullscreen_btn.setIcon(self.fullscreen_icon_fullscreen) | |
| 380 self.showFullScreen() | |
| 381 else: | |
| 382 self.fullscreen_btn.setIcon(self.fullscreen_icon_normal) | |
| 383 self.showNormal() | |
| 384 | |
|
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
385 def closeEvent(self, a0: QCloseEvent|None) -> None: |
| 4206 | 386 super().closeEvent(a0) |
| 387 global running | |
| 388 running = False | |
| 389 | |
| 390 def toggle_video(self): | |
| 391 assert self.webrtc_call is not None | |
| 392 self.webrtc_call.webrtc.video_muted = not self.webrtc_call.webrtc.video_muted | |
| 393 self.toggle_video_btn.activated = not self.webrtc_call.webrtc.video_muted | |
| 394 | |
| 395 def toggle_audio(self): | |
| 396 assert self.webrtc_call is not None | |
| 397 self.webrtc_call.webrtc.audio_muted = not self.webrtc_call.webrtc.audio_muted | |
| 398 self.toggle_audio_btn.activated = not self.webrtc_call.webrtc.audio_muted | |
| 399 | |
| 400 def share_desktop(self): | |
| 401 assert self.webrtc_call is not None | |
| 402 if self.webrtc_call.webrtc.desktop_sharing: | |
| 403 self.webrtc_call.webrtc.desktop_sharing = False | |
| 404 self.share_desktop_btn.activated = False | |
| 405 elif display_servers.detect() == display_servers.X11: | |
| 406 aio.run_async(self.show_X11_screen_dialog()) | |
| 407 else: | |
| 408 self.webrtc_call.webrtc.desktop_sharing = True | |
| 409 | |
| 410 def hang_up(self): | |
| 411 self.close() | |
| 412 | |
|
4210
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
413 @staticmethod |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
414 def can_run(): |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
415 # if a known display server is detected, we should be able to run |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
416 return display_servers.detect() is not None |
|
9218d4331bb2
cli (call): `tui` output implementation:
Goffi <goffi@goffi.org>
parents:
4206
diff
changeset
|
417 |
| 4206 | 418 async def show_X11_screen_dialog(self): |
| 419 assert self.webrtc_call is not None | |
| 420 windows_data = display_servers.x11_list_windows() | |
| 421 dialog = X11DesktopScreenDialog(windows_data, self) | |
| 422 selected = await dialog.a_show() | |
| 423 if selected is not None: | |
| 424 xid = selected["id"] | |
| 425 self.webrtc_call.webrtc.desktop_sharing_data = {"xid": xid} | |
| 426 self.webrtc_call.webrtc.desktop_sharing = True | |
| 427 self.share_desktop_btn.activated = True |