xdp screen cast example, modified to pipe to ffmpeg

stream.sh

mkfifo /tmp/gfifo

./xdp-screen-cast-ffmpeg.py &  # or run in another terminal
# Note the gstreamer pipeline
# 'pipewiresrc fd=%d path=%u ! videorate ! video/x-raw,framerate=60/1 ! videoconvert ! filesink location=/tmp/gfifo'
# I specified a 60/1 frameerate. You can do other pre-processing here, or offload it to ffmpeg.

# Encode the stream and pipe to netcat
# note: must specify correct resolution here
# we can now use h264_nvenc, mpegts, and anything else ffmpeg has
ffmpeg -f rawvideo -pixel_format bgra -video_size 2560x1600 -i /tmp/gfifo \
    -c:v h264_nvenc -preset:v llhq -tune ull -zerolatency 1 -delay 0 -f mpegts - \
    | nc -l -p 9001

# on remote system do
nc $ip_of_server 9001 | mpv --profile=low-latency --untimed --no-demuxer-thread -

xdp-screen-cast-ffmpeg.py

#!/usr/bin/python3

# Source from: https://gitlab.gnome.org/-/snippets/19

import re
import signal
import dbus
from gi.repository import GLib
from dbus.mainloop.glib import DBusGMainLoop

import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst

DBusGMainLoop(set_as_default=True)
Gst.init(None)

loop = GLib.MainLoop()

bus = dbus.SessionBus()
request_iface = 'org.freedesktop.portal.Request'
screen_cast_iface = 'org.freedesktop.portal.ScreenCast'

pipeline = None

def terminate():
    if pipeline is not None:
        self.player.set_state(Gst.State.NULL)
    loop.quit()

request_token_counter = 0
session_token_counter = 0
sender_name = re.sub(r'\.', r'_', bus.get_unique_name()[1:])

def new_request_path():
    global request_token_counter
    request_token_counter = request_token_counter + 1
    token = 'u%d'%request_token_counter
    path = '/org/freedesktop/portal/desktop/request/%s/%s'%(sender_name, token)
    return (path, token)

def new_session_path():
    global session_token_counter
    session_token_counter = session_token_counter + 1
    token = 'u%d'%session_token_counter
    path = '/org/freedesktop/portal/desktop/session/%s/%s'%(sender_name, token)
    return (path, token)

def screen_cast_call(method, callback, *args, options={}):
    (request_path, request_token) = new_request_path()
    bus.add_signal_receiver(callback,
                            'Response',
                            request_iface,
                            'org.freedesktop.portal.Desktop',
                            request_path)
    options['handle_token'] = request_token
    method(*(args + (options, )),
           dbus_interface=screen_cast_iface)

def on_gst_message(bus, message):
    type = message.type
    if type == Gst.MessageType.EOS or type == Gst.MessageType.ERROR:
        terminate()

def play_pipewire_stream(node_id):
    empty_dict = dbus.Dictionary(signature="sv")
    fd_object = portal.OpenPipeWireRemote(session, empty_dict,
                                          dbus_interface=screen_cast_iface)
    fd = fd_object.take()
    pipeline = Gst.parse_launch('pipewiresrc fd=%d path=%u ! videorate ! video/x-raw,framerate=60/1 ! videoconvert ! filesink location=/tmp/gfifo' %(fd, node_id))
    pipeline.set_state(Gst.State.PLAYING)
    pipeline.get_bus().connect('message', on_gst_message)

def on_start_response(response, results):
    if response != 0:
        print("Failed to start: %s"%response)
        terminate()
        return

    print("streams:")
    for (node_id, stream_properties) in results['streams']:
        print("stream {}".format(node_id))
        play_pipewire_stream(node_id)

def on_select_sources_response(response, results):
    if response != 0:
        print("Failed to select sources: %d"%response)
        terminate()
        return

    print("sources selected")
    global session
    screen_cast_call(portal.Start, on_start_response,
                     session, '')

def on_create_session_response(response, results):
    if response != 0:
        print("Failed to create session: %d"%response)
        terminate()
        return

    global session
    session = results['session_handle']
    print("session %s created"%session)

    screen_cast_call(portal.SelectSources, on_select_sources_response,
                     session,
                     options={ 'multiple': False,
                               'types': dbus.UInt32(1|2) })

portal = bus.get_object('org.freedesktop.portal.Desktop',
                             '/org/freedesktop/portal/desktop')

(session_path, session_token) = new_session_path()
screen_cast_call(portal.CreateSession, on_create_session_response,
                 options={ 'session_handle_token': session_token })

try:
    loop.run()
except KeyboardInterrupt:
    terminate()

Note - anyone without an nvidia card, change h264_nvenc to plain h264.

For anyone looking for local playback, you can omit ffmpeg and mpv and simply run ffplay -f rawvideo -pixel_format bgra -video_size 1920x1080 -i /tmp/gfifo. I get a bit of latency still, but less than encoding and decoding. Apparently there are also -fflags nobuffer, or -tune zerolatency, these caused some problems for me and haven't yet looked into other flags to tune it towards live playback of raw video.

Note to self: add a way to get the screen resolution and give this to ffplay automatically.

I'm not sure why this no longer works for me. Whether trying mpv or ffplay, I get the first image, and maybe a very lagged out blur of movement, but then it's all frozen.

By the way, debugging output can be enabled by adding this after the import of Gst, GObject:

from gi.repository import GObject, Gst

Gst.debug_set_active(True)
Gst.debug_set_default_threshold(4)
GObject.threads_init()

Also hopefully this gist is no longer needed soon. There's been work on a "pipewiregrab" patch for ffmpeg https://ffmpeg.org/pipermail/ffmpeg-devel/2024-August/thread.html#331913

Took a look at why this stopped working for me again... really dumb but an extra space between the mpv args seems to create havoc.

For debugging I send the stream to a file

pipeline = Gst.parse_launch('pipewiresrc fd=%d path=%u ! videorate ! video/x-raw,framerate=60/1 ! videoconvert ! filesink location=/home/andy/xdp-video.raw' %(fd, node_id))

and now I can run it with mpv with

mpv --demuxer=rawvideo --demuxer-rawvideo-w=2560 --demuxer-rawvideo-h=1440 --demuxer-rawvideo-format=bgra --demuxer-rawvideo-fps=60 --demuxer-rawvideo-mp-format=bgra --profile=low-latency --untimed --no-demuxer-thread ~/xdp-video.raw

or ffplay per stellarpower's comment above.

But when streaming with the fifo, ffplay seems pretty bad like 1 frame every 5 seconds. Streaming with mpv, better but still very laggy. Maybe related to my gpu setup will have to poke at it.