libervia-backend: libervia/frontends/tools/webrtc.py comparison

comparison libervia/frontends/tools/webrtc.py @ 4209:fe29fbdabce6

frontends (tools/webrtc): add a options to merge video for PiP and to specify size: - the `merge_pip` option is now used to indicate if local feedback and remote video streams must be merged with local feedback being a PiP of remote video. By default, it's done for `SINKS_AUTO`, but it can be manually specified to use it with `SINKS_APP` or to split ``SINKS_AUTO`` in 2 windows. - target size of the compositor used with `merge_pip` can be specified. By default, 720p is used. rel 428

author	Goffi <goffi@goffi.org>
date	Fri, 16 Feb 2024 18:46:02 +0100
parents	17a8168966f9
children	d01b8d002619

comparison

equal deleted inserted replaced

-:fd9e78b7a0cd
+:fe29fbdabce6
 @dataclass
 class AppSinkData:
 local_video_cb: Callable
-remote_video_cb: Callable
+remote_video_cb: Callable|None
 class DesktopPortal:
 def __init__(self, webrtc: "WebRTC"):
 profile: str,
 sources: str = SOURCES_AUTO,
 sinks: str = SINKS_AUTO,
 appsink_data: AppSinkData | None = None,
 reset_cb: Callable | None = None,
+merge_pip: bool|None = None,
+target_size: tuple[int, int]|None = None,
 ) -> None:
+"""Initializes a new WebRTC instance.
+@param bridge: An instance of backend bridge.
+@param profile: Libervia profile.
+@param sources: Which kind of source to use.
+@param sinks: Which kind of sinks to use.
+@param appsink_data: configuration data for appsink (when SINKS_APP is used). Must
+not be used for other sinks.
+@param reset_cb: An optional Callable that is triggered on reset events. Can be
+used to reset UI data on new calls.
+@param merge_pip: A boolean flag indicating whether Picture-in-Picture mode is
+enabled. When PiP is used, local feedback is merged to remote video stream.
+Only one video stream is then produced (the local one).
+If None, PiP mode is selected automatically according to selected sink (it's
+used for SINKS_AUTO only for now).
+@param target_size: Expected size of the final sink stream. Mainly use by composer
+when ``merge_pip`` is set.
+None to autodetect (not real autodetection implemeted yet, default to
+(1280,720)).
+"""
 self.main_loop = asyncio.get_event_loop()
 self.bridge = bridge
 self.profile = profile
 self.pipeline = None
 self._audio_muted = False
 self._video_muted = False
 self._desktop_sharing = False
 self.desktop_sharing_data = None
 self.sources = sources
 self.sinks = sinks
+if target_size is None:
+target_size=(1280, 720)
+self.target_width, self.target_height = target_size
+if merge_pip is None:
+merge_pip = sinks == SINKS_AUTO
+self.merge_pip = merge_pip
 if sinks == SINKS_APP:
+if (
+merge_pip
+and appsink_data is not None
+and appsink_data.remote_video_cb is not None
+):
+raise ValueError("Remote_video_cb can't be used when merge_pip is used!")
 self.appsink_data = appsink_data
 elif appsink_data is not None:
 raise exceptions.InternalError(
 "appsink_data can only be used for SINKS_APP sinks"
 )
 video_source_elt = "videotestsrc is-live=true pattern=ball"
 audio_source_elt = "audiotestsrc"
 else:
 raise exceptions.InternalError(f'Unknown "sources" value: {self.sources!r}')
-extra_elt = ""
 if self.sinks == SINKS_APP:
 local_video_sink_elt = (
 "appsink name=local_video_sink emit-signals=true drop=true max-buffers=1 "
 "sync=True"
 )
 elif self.sinks == SINKS_AUTO:
-extra_elt = "compositor name=compositor ! autovideosink"
+local_video_sink_elt = "autovideosink"
-local_video_sink_elt = """compositor.sink_1"""
 else:
 raise exceptions.InternalError(f"Unknown sinks value {self.sinks!r}")
+if self.merge_pip:
+extra_elt = (
+"compositor name=compositor background=black "
+f"! video/x-raw,width={self.target_width},height={self.target_height},"
+"framerate=30/1 "
+f"! {local_video_sink_elt}"
+)
+local_video_sink_elt = "compositor.sink_1"
+else:
+extra_elt = ""
 self.gst_pipe_desc = f"""
 webrtcbin latency=100 name=sendrecv bundle-policy=max-bundle
 input-selector name=video_selector
 """
 log.debug(f"Gstreamer pipeline: {self.gst_pipe_desc}")
 # Create the pipeline
-self.pipeline = Gst.parse_launch(self.gst_pipe_desc)
+try:
+self.pipeline = Gst.parse_launch(self.gst_pipe_desc)
+except Exception:
+log.exception("Can't parse pipeline")
+self.pipeline = None
 if not self.pipeline:
 raise exceptions.InternalError("Failed to create Gstreamer pipeline.")
 self.webrtcbin = self.pipeline.get_by_name("sendrecv")
 self.video_src = self.pipeline.get_by_name("video_src")
 log.debug("===> VIDEO OK")
 self._remote_video_pad = pad
 # Check and log the original size of the video
-width = s.get_int("width").value
+width = self.target_width
-height = s.get_int("height").value
+height = self.target_height
 log.info(f"Original video size: {width}x{height}")
 # This is a fix for an issue found with Movim on desktop: a non standard
 # resolution is used (990x557) resulting in bad alignement and no color in
 # rendered image
 width = (width + 3) // 4 * 4
 height = (height + 3) // 4 * 4
 log.info(f"Adjusted video size: {width}x{height}")
 conv = Gst.ElementFactory.make("videoconvert")
-if self.sinks == SINKS_APP:
+if self.merge_pip:
+# with ``merge_pip`` set, we plug the remote stream to the composer
+compositor = self.pipeline.get_by_name("compositor")
+sink1_pad = compositor.get_static_pad("sink_1")
+local_width, local_height = self.scaled_dimensions(
+sink1_pad.get_property("width"),
+sink1_pad.get_property("height"),
+width // 3,
+height // 3,
+)
+sink1_pad.set_property("xpos", width - local_width)
+sink1_pad.set_property("ypos", height - local_height)
+sink1_pad.set_property("width", local_width)
+sink1_pad.set_property("height", local_height)
+sink1_pad.set_property("sizing-policy", 1)
+sink1_pad.set_property("zorder", 1)
+# Request a new pad for the remote stream
+sink_pad_template = compositor.get_pad_template("sink_%u")
+remote_video_sink = compositor.request_pad(sink_pad_template, None, None)
+remote_video_sink.set_property("zorder", 0)
+remote_video_sink.set_property("width", width)
+remote_video_sink.set_property("height", height)
+remote_video_sink.set_property("sizing-policy", 1)
+elif self.sinks == SINKS_APP:
+# ``app`` sink without ``self.merge_pip`` set, be create the sink and
+# connect it to the ``remote_video_cb``.
 assert self.appsink_data is not None
 remote_video_sink = Gst.ElementFactory.make("appsink")
-appsink_caps = Gst.Caps.from_string("video/x-raw,format=RGB")
+remote_video_caps = Gst.Caps.from_string("video/x-raw,format=RGB")
-remote_video_sink.set_property("caps", appsink_caps)
+remote_video_sink.set_property("caps", remote_video_caps)
 remote_video_sink.set_property("emit-signals", True)
 remote_video_sink.set_property("drop", True)
 remote_video_sink.set_property("max-buffers", 1)
 remote_video_sink.set_property("sync", True)
 remote_video_sink.connect("new-sample", self.appsink_data.remote_video_cb)
 self.pipeline.add(remote_video_sink)
 elif self.sinks == SINKS_AUTO:
-compositor = self.pipeline.get_by_name("compositor")
+# if ``self.merge_pip`` is not set, we create a dedicated
+# ``autovideosink`` for remote stream.
-sink1_pad = compositor.get_static_pad("sink_1")
+remote_video_sink = Gst.ElementFactory.make("autovideosink")
+self.pipeline.add(remote_video_sink)
-local_width, local_height = self.scaled_dimensions(
-sink1_pad.get_property("width"),
-sink1_pad.get_property("height"),
-width // 3,
-height // 3,
-)
-sink1_pad.set_property("xpos", width - local_width)
-sink1_pad.set_property("ypos", height - local_height)
-sink1_pad.set_property("width", local_width)
-sink1_pad.set_property("height", local_height)
-sink1_pad.set_property("zorder", 1)
-# Request a new pad for the remote stream
-sink_pad_template = compositor.get_pad_template("sink_%u")
-remote_video_sink = compositor.request_pad(sink_pad_template, None, None)
-remote_video_sink.set_property("zorder", 0)
 else:
 raise exceptions.InternalError(f'Unhandled "sinks" value: {self.sinks!r}')
 if adjust_resolution:
 videoscale = Gst.ElementFactory.make("videoscale")

Mercurial > libervia-backend

comparison libervia/frontends/tools/webrtc.py @ 4209:fe29fbdabce6