Mercurial > libervia-backend

comparison libervia/frontends/tools/webrtc.py @ 4205:17a8168966f9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
frontends (tools/webrtc): implement screensharing for Wayland + bug fixes: - Freedesktop Desktop Screenshare port is now used when Wayland is detected (needs `xdg-desktop-portal` with the implementation corresponding to desktop environment). - Add a binding feature to feedback state to application (e.g. if desktop sharing is cancelled from desktop environment, or at portal's permission request level). - fix misnaming of video source (was wrongly named `camera` instead of `video`). - fix desktop sharing pad selection in `input-selector` when it has been added once, then removed, then added again. rel 434
author Goffi <goffi@goffi.org>
date Thu, 18 Jan 2024 23:29:25 +0100
parents 879bad48cc2d
children fe29fbdabce6
comparison
equal deleted inserted replaced
4204:879bad48cc2d 4205:17a8168966f9
34 ) 34 )
35 import asyncio 35 import asyncio
36 from dataclasses import dataclass 36 from dataclasses import dataclass
37 from datetime import datetime 37 from datetime import datetime
38 import logging 38 import logging
39 from random import randint
39 import re 40 import re
40 from typing import Callable 41 from typing import Callable
41 from urllib.parse import quote_plus 42 from urllib.parse import quote_plus
42 43
43 from libervia.backend.tools.common import data_format 44 from libervia.backend.tools.common import data_format
59 SINKS_APP = "app" 60 SINKS_APP = "app"
60 SINKS_AUTO = "auto" 61 SINKS_AUTO = "auto"
61 SINKS_TEST = "test" 62 SINKS_TEST = "test"
62 63
63 64
65 class ScreenshareError(Exception):
66 pass
67
68
64 @dataclass 69 @dataclass
65 class AppSinkData: 70 class AppSinkData:
66 local_video_cb: Callable 71 local_video_cb: Callable
67 remote_video_cb: Callable 72 remote_video_cb: Callable
73
74
75 class DesktopPortal:
76
77 def __init__(self, webrtc: "WebRTC"):
78 import dbus
79 from dbus.mainloop.glib import DBusGMainLoop
80 # we want monitors + windows, see https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.ScreenCast.html#org-freedesktop-portal-screencast-availablesourcetypes
81 self.dbus = dbus
82 self.webrtc = webrtc
83 self.sources_type = dbus.UInt32(7)
84 DBusGMainLoop(set_as_default=True)
85 self.session_bus = dbus.SessionBus()
86 portal_object = self.session_bus.get_object(
87 'org.freedesktop.portal.Desktop',
88 '/org/freedesktop/portal/desktop'
89 )
90 self.screencast_interface = dbus.Interface(
91 portal_object,
92 'org.freedesktop.portal.ScreenCast'
93 )
94 self.session_interface = None
95 self.session_signal = None
96 self.handle_counter = 0
97 self.session_handle = None
98 self.stream_data: dict|None = None
99
100 @property
101 def handle_token(self):
102 self.handle_counter += 1
103 return f"libervia{self.handle_counter}"
104
105 def on_session_closed(self, details: dict) -> None:
106 if self.session_interface is not None:
107 self.session_interface = None
108 self.webrtc.desktop_sharing = False
109 if self.session_signal is not None:
110 self.session_signal.remove()
111 self.session_signal = None
112
113
114 async def dbus_call(self, method_name: str, *args) -> dict:
115 """Call a screenshare portal method
116
117 This method handle the signal response.
118 @param method_name: method to call
119 @param args: extra args
120 `handle_token` will be automatically added to the last arg (option dict)
121 @return: method result
122 """
123 if self.session_handle is not None:
124 self.end_screenshare()
125 method = getattr(self.screencast_interface, method_name)
126 options = args[-1]
127 reply_fut = asyncio.Future()
128 signal_fut = asyncio.Future()
129 # cf. https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.Request.html
130 handle_token = self.handle_token
131 sender = self.session_bus.get_unique_name().replace(".", "_")[1:]
132 path = f"/org/freedesktop/portal/desktop/request/{sender}/{handle_token}"
133 signal_match = None
134
135 def on_signal(response, results):
136 assert signal_match is not None
137 signal_match.remove()
138 if response == 0:
139 signal_fut.set_result(results)
140 elif response == 1:
141 signal_fut.set_exception(
142 exceptions.CancelError("Cancelled by user.")
143 )
144 else:
145 signal_fut.set_exception(ScreenshareError(
146 f"Can't get signal result"
147 ))
148
149 signal_match = self.session_bus.add_signal_receiver(
150 on_signal,
151 signal_name="Response",
152 dbus_interface="org.freedesktop.portal.Request",
153 path=path
154 )
155
156 options["handle_token"] = handle_token
157
158 method(
159 *args,
160 reply_handler=reply_fut.set_result,
161 error_handler=reply_fut.set_exception
162 )
163 try:
164 await reply_fut
165 except Exception as e:
166 raise ScreenshareError(f"Can't ask screenshare permission: {e}")
167 return await signal_fut
168
169 async def request_screenshare(self) -> dict:
170 session_data = await self.dbus_call(
171 "CreateSession",
172 {
173 "session_handle_token": str(randint(1, 2**32)),
174 }
175 )
176 try:
177 session_handle = session_data["session_handle"]
178 except KeyError:
179 raise ScreenshareError("Can't get session handle")
180 self.session_handle = session_handle
181
182
183 await self.dbus_call(
184 "SelectSources",
185 session_handle,
186 {
187 "multiple": True,
188 "types": self.sources_type,
189 "modal": True
190 }
191 )
192 screenshare_data = await self.dbus_call(
193 "Start",
194 session_handle,
195 "",
196 {}
197 )
198
199 session_object = self.session_bus.get_object(
200 'org.freedesktop.portal.Desktop',
201 session_handle
202 )
203 self.session_interface = self.dbus.Interface(
204 session_object,
205 'org.freedesktop.portal.Session'
206 )
207
208 self.session_signal = self.session_bus.add_signal_receiver(
209 self.on_session_closed,
210 signal_name="Closed",
211 dbus_interface="org.freedesktop.portal.Session",
212 path=session_handle
213 )
214
215 try:
216 node_id, stream_data = screenshare_data["streams"][0]
217 source_type = int(stream_data["source_type"])
218 except (IndexError, KeyError):
219 raise ScreenshareError("Can't parse stream data")
220 self.stream_data = stream_data = {
221 "session_handle": session_handle,
222 "node_id": node_id,
223 "source_type": source_type
224 }
225 try:
226 height = int(stream_data["size"][0])
227 weight = int(stream_data["size"][1])
228 except (IndexError, KeyError):
229 pass
230 else:
231 stream_data["size"] = (height, weight)
232
233 return self.stream_data
234
235 def end_screenshare(self) -> None:
236 """Close a running screenshare session, if any."""
237 if self.session_interface is None:
238 return
239 self.session_interface.Close()
240 self.on_session_closed({})
68 241
69 242
70 class WebRTC: 243 class WebRTC:
71 """GSTreamer based WebRTC implementation for audio and video communication. 244 """GSTreamer based WebRTC implementation for audio and video communication.
72 245
98 elif appsink_data is not None: 271 elif appsink_data is not None:
99 raise exceptions.InternalError( 272 raise exceptions.InternalError(
100 "appsink_data can only be used for SINKS_APP sinks" 273 "appsink_data can only be used for SINKS_APP sinks"
101 ) 274 )
102 self.reset_cb = reset_cb 275 self.reset_cb = reset_cb
276 if current_server == display_servers.WAYLAND:
277 self.desktop_portal = DesktopPortal(self)
278 else:
279 self.desktop_portal = None
103 self.reset_instance() 280 self.reset_instance()
104 281
105 @property 282 @property
106 def audio_muted(self): 283 def audio_muted(self):
107 return self._audio_muted 284 return self._audio_muted
129 @desktop_sharing.setter 306 @desktop_sharing.setter
130 def desktop_sharing(self, active: bool) -> None: 307 def desktop_sharing(self, active: bool) -> None:
131 if active != self._desktop_sharing: 308 if active != self._desktop_sharing:
132 self._desktop_sharing = active 309 self._desktop_sharing = active
133 self.on_desktop_switch(active) 310 self.on_desktop_switch(active)
311 try:
312 cb = self.bindings["desktop_sharing"]
313 except KeyError:
314 pass
315 else:
316 cb(active)
134 317
135 @property 318 @property
136 def sdp_set(self): 319 def sdp_set(self):
137 return self._sdp_set 320 return self._sdp_set
138 321
158 @property 341 @property
159 def media_types_inv(self) -> dict: 342 def media_types_inv(self) -> dict:
160 if self._media_types_inv is None: 343 if self._media_types_inv is None:
161 raise Exception("self._media_types_inv should not be None!") 344 raise Exception("self._media_types_inv should not be None!")
162 return self._media_types_inv 345 return self._media_types_inv
346
347 def bind(self, **kwargs: Callable) -> None:
348 self.bindings.clear()
349 for key, cb in kwargs.items():
350 if key not in ("desktop_sharing",):
351 raise ValueError(
352 'Only "desktop_sharing" is currently allowed for binding'
353 )
354 self.bindings[key] = cb
355
163 356
164 def generate_dot_file( 357 def generate_dot_file(
165 self, 358 self,
166 filename: str = "pipeline", 359 filename: str = "pipeline",
167 details: Gst.DebugGraphDetails = Gst.DebugGraphDetails.ALL, 360 details: Gst.DebugGraphDetails = Gst.DebugGraphDetails.ALL,
339 } 532 }
340 self._media_types = None 533 self._media_types = None
341 self._media_types_inv = None 534 self._media_types_inv = None
342 self.audio_valve = None 535 self.audio_valve = None
343 self.video_valve = None 536 self.video_valve = None
537 if self.desktop_portal is not None:
538 self.desktop_portal.end_screenshare()
539 self.desktop_sharing = False
540 self.desktop_sink_pad = None
541 self.bindings = {}
344 if self.reset_cb is not None: 542 if self.reset_cb is not None:
345 self.reset_cb() 543 self.reset_cb()
346 544
347 545
348 async def setup_call( 546 async def setup_call(
920 """Handles (un)muting of video. 1118 """Handles (un)muting of video.
921 1119
922 @param muted: True if video is muted. 1120 @param muted: True if video is muted.
923 """ 1121 """
924 if self.video_selector is not None: 1122 if self.video_selector is not None:
925 current_source = None if muted else "desktop" if self.desktop_sharing else "camera" 1123 current_source = None if muted else "desktop" if self.desktop_sharing else "video"
926 self.switch_video_source(current_source) 1124 self.switch_video_source(current_source)
927 state = "muted" if muted else "unmuted" 1125 state = "muted" if muted else "unmuted"
928 log.info(f"Video is now {state}") 1126 log.info(f"Video is now {state}")
929 1127
930 def on_desktop_switch(self, desktop_active: bool) -> None: 1128 def on_desktop_switch(self, desktop_active: bool) -> None:
931 """Switches the video source between desktop and camera. 1129 """Switches the video source between desktop and video.
932 1130
933 @param desktop_active: True if desktop must be active. False for camera. 1131 @param desktop_active: True if desktop must be active. False for video.
934 """ 1132 """
1133 if desktop_active and self.desktop_portal is not None:
1134 aio.run_async(self.on_desktop_switch_portal(desktop_active))
1135 else:
1136 self.do_desktop_switch(desktop_active)
1137
1138 async def on_desktop_switch_portal(self, desktop_active: bool) -> None:
1139 """Call freedesktop screenshare portal and the activate the shared stream"""
1140 assert self.desktop_portal is not None
1141 try:
1142 screenshare_data = await self.desktop_portal.request_screenshare()
1143 except exceptions.CancelError:
1144 self.desktop_sharing = False
1145 return
1146 self.desktop_sharing_data = {
1147 "path": str(screenshare_data["node_id"])
1148 }
1149 self.do_desktop_switch(desktop_active)
1150
1151 def do_desktop_switch(self, desktop_active: bool) -> None:
935 if self.video_muted: 1152 if self.video_muted:
936 # Update the active source state but do not switch 1153 # Update the active source state but do not switch
937 self.desktop_sharing = desktop_active 1154 self.desktop_sharing = desktop_active
938 return 1155 return
939 1156
940 source = "desktop" if desktop_active else "camera" 1157 source = "desktop" if desktop_active else "video"
941 self.switch_video_source(source) 1158 self.switch_video_source(source)
942 self.desktop_sharing = desktop_active 1159 self.desktop_sharing = desktop_active
943 1160
944 def switch_video_source(self, source: str|None) -> None: 1161 def switch_video_source(self, source: str|None) -> None:
945 """Activates the specified source while deactivating the others. 1162 """Activates the specified source while deactivating the others.
946 1163
947 @param source: 'desktop', 'camera', 'muted' or None for muted source. 1164 @param source: 'desktop', 'video', 'muted' or None for muted source.
948 """ 1165 """
949 if source is None: 1166 if source is None:
950 source = "muted" 1167 source = "muted"
951 if source not in ["camera", "muted", "desktop"]: 1168 if source not in ["video", "muted", "desktop"]:
952 raise ValueError( 1169 raise ValueError(
953 f"Invalid source: {source!r}, use one of {'camera', 'muted', 'desktop'}" 1170 f"Invalid source: {source!r}, use one of {'video', 'muted', 'desktop'}"
954 ) 1171 )
955 1172
956 self.pipeline.set_state(Gst.State.PAUSED) 1173 self.pipeline.set_state(Gst.State.PAUSED)
957 1174
958 # Create a new desktop source if necessary 1175 # Create a new desktop source if necessary
959 if source == "desktop": 1176 if source == "desktop":
960 self._setup_desktop_source(self.desktop_sharing_data) 1177 self._setup_desktop_source(self.desktop_sharing_data)
961 1178
962 # Activate the chosen source and deactivate the others 1179 # Activate the chosen source and deactivate the others
963 for src_name in ["camera", "muted", "desktop"]: 1180 for src_name in ["video", "muted", "desktop"]:
964 src_element = self.pipeline.get_by_name(f"{src_name}_src") 1181 src_element = self.pipeline.get_by_name(f"{src_name}_src")
965 if src_name == source: 1182 if src_name == source:
966 if src_element: 1183 if src_element:
967 src_element.set_state(Gst.State.PLAYING) 1184 src_element.set_state(Gst.State.PLAYING)
968 else: 1185 else:
969 if src_element: 1186 if src_element:
970 src_element.set_state(Gst.State.NULL) 1187 if src_name == "desktop":
971 if src_name == "desktop": 1188 self._remove_desktop_source(src_element)
972 self._remove_desktop_source(src_element) 1189 else:
973 1190 src_element.set_state(Gst.State.NULL)
974 # Set the video_selector active pad 1191
975 pad_name = f"sink_{['camera', 'muted', 'desktop'].index(source)}" 1192 # Set the video_selector active pad
976 pad = self.video_selector.get_static_pad(pad_name) 1193 if source == "desktop":
977 self.video_selector.props.active_pad = pad 1194 if self.desktop_sink_pad:
978 self.pipeline.set_state(Gst.State.PLAYING) 1195 pad = self.desktop_sink_pad
1196 else:
1197 log.error(f"No desktop pad available")
1198 pad = None
1199 else:
1200 pad_name = f"sink_{['video', 'muted'].index(source)}"
1201 pad = self.video_selector.get_static_pad(pad_name)
1202
1203 if pad is not None:
1204 self.video_selector.props.active_pad = pad
1205
1206 self.pipeline.set_state(Gst.State.PLAYING)
979 1207
980 def _setup_desktop_source(self, properties: dict[str, object]|None) -> None: 1208 def _setup_desktop_source(self, properties: dict[str, object]|None) -> None:
981 """Set up a new desktop source. 1209 """Set up a new desktop source.
982 1210
983 @param properties: The properties to set on the desktop source. 1211 @param properties: The properties to set on the desktop source.
984 """ 1212 """
985 desktop_src = Gst.ElementFactory.make("ximagesrc", "desktop_src") 1213 source_elt = "ximagesrc" if self.desktop_portal is None else "pipewiresrc"
1214 desktop_src = Gst.ElementFactory.make(source_elt, "desktop_src")
986 if properties is None: 1215 if properties is None:
987 properties = {} 1216 properties = {}
988 for key, value in properties.items(): 1217 for key, value in properties.items():
989 log.debug(f"setting ximagesrc property: {key!r}={value!r}") 1218 log.debug(f"setting {source_elt} property: {key!r}={value!r}")
990 desktop_src.set_property(key, value) 1219 desktop_src.set_property(key, value)
991 video_convert = Gst.ElementFactory.make("videoconvert", "desktop_videoconvert") 1220 video_convert = Gst.ElementFactory.make("videoconvert", "desktop_videoconvert")
992 queue = Gst.ElementFactory.make("queue", "desktop_queue") 1221 queue = Gst.ElementFactory.make("queue", "desktop_queue")
993 queue.set_property("leaky", "downstream") 1222 queue.set_property("leaky", "downstream")
994 1223
998 1227
999 desktop_src.link(video_convert) 1228 desktop_src.link(video_convert)
1000 video_convert.link(queue) 1229 video_convert.link(queue)
1001 1230
1002 sink_pad_template = self.video_selector.get_pad_template("sink_%u") 1231 sink_pad_template = self.video_selector.get_pad_template("sink_%u")
1003 sink_pad = self.video_selector.request_pad(sink_pad_template, None, None) 1232 self.desktop_sink_pad = self.video_selector.request_pad(sink_pad_template, None, None)
1004 queue_src_pad = queue.get_static_pad("src") 1233 queue_src_pad = queue.get_static_pad("src")
1005 queue_src_pad.link(sink_pad) 1234 queue_src_pad.link(self.desktop_sink_pad)
1006 1235
1007 desktop_src.sync_state_with_parent() 1236 desktop_src.sync_state_with_parent()
1008 video_convert.sync_state_with_parent() 1237 video_convert.sync_state_with_parent()
1009 queue.sync_state_with_parent() 1238 queue.sync_state_with_parent()
1010 1239
1014 @param desktop_src: The desktop source to remove. 1243 @param desktop_src: The desktop source to remove.
1015 """ 1244 """
1016 # Remove elements for the desktop source 1245 # Remove elements for the desktop source
1017 video_convert = self.pipeline.get_by_name("desktop_videoconvert") 1246 video_convert = self.pipeline.get_by_name("desktop_videoconvert")
1018 queue = self.pipeline.get_by_name("desktop_queue") 1247 queue = self.pipeline.get_by_name("desktop_queue")
1248
1019 if video_convert: 1249 if video_convert:
1020 video_convert.set_state(Gst.State.NULL) 1250 video_convert.set_state(Gst.State.NULL)
1021 desktop_src.unlink(video_convert) 1251 desktop_src.unlink(video_convert)
1022 self.pipeline.remove(video_convert) 1252 self.pipeline.remove(video_convert)
1253
1023 if queue: 1254 if queue:
1024 queue.set_state(Gst.State.NULL) 1255 queue.set_state(Gst.State.NULL)
1025 self.pipeline.remove(queue) 1256 self.pipeline.remove(queue)
1257
1258 desktop_src.set_state(Gst.State.NULL)
1026 self.pipeline.remove(desktop_src) 1259 self.pipeline.remove(desktop_src)
1260
1261 # Release the pad associated with the desktop source
1262 if self.desktop_sink_pad:
1263 self.video_selector.release_request_pad(self.desktop_sink_pad)
1264 self.desktop_sink_pad = None
1265
1266 if self.desktop_portal is not None:
1267 self.desktop_portal.end_screenshare()
1027 1268
1028 async def end_call(self) -> None: 1269 async def end_call(self) -> None:
1029 """Stop streaming and clean instance""" 1270 """Stop streaming and clean instance"""
1030 self.reset_instance() 1271 self.reset_instance()