vdpau: add working decoder device module (fixes #26385)

This associates the correct X11 screen and does not depend on GL for no
particular reasons.

modules/hw/vdpau/Makefile.am

@@ -8,6 +8,11 @@ libvlc_vdpau_la_LDFLAGS = \
 	-export-symbols-regex ^vdp_ \
 	-version-info 0:0:0
 
+libvdpau_instance_plugin_la_SOURCES = hw/vdpau/device.c
+libvdpau_instance_plugin_la_CFLAGS = $(AM_CFLAGS) $(X_CFLAGS) $(VDPAU_CFLAGS)
+libvdpau_instance_plugin_la_LIBADD = libvlc_vdpau.la \
+	$(X_LIBS) $(X_PRE_LIBS) -lX11
+
 libvdpau_avcodec_plugin_la_SOURCES = hw/vdpau/avcodec.c hw/vdpau/picture.c
 libvdpau_avcodec_plugin_la_CFLAGS = $(AM_CFLAGS) \
 	$(X_CFLAGS) $(AVCODEC_CFLAGS) $(VDPAU_CFLAGS)
@@ -38,7 +43,8 @@ libvdpau_display_plugin_la_LIBADD = libvlc_vdpau.la libvlc_xcb_events.la \
 
 if HAVE_VDPAU
 pkglib_LTLIBRARIES += libvlc_vdpau.la
-vdpau_LTLIBRARIES = libvdpau_deinterlace_plugin.la libvdpau_adjust_plugin.la \
+vdpau_LTLIBRARIES = libvdpau_instance_plugin.la \
+	libvdpau_deinterlace_plugin.la libvdpau_adjust_plugin.la \
 	libvdpau_sharpen_plugin.la libvdpau_chroma_plugin.la
 if HAVE_XCB
 vdpau_LTLIBRARIES += libvdpau_display_plugin.la

modules/hw/vdpau/device.c

@@ -0,0 +1,125 @@
+/*****************************************************************************
+ * device.c: VDPAU instance management for VLC
+ *****************************************************************************
+ * Copyright (C) 2013-2021 Rémi Denis-Courmont
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include <vlc_common.h>
+#include <vlc_codec.h>
+#include <vlc_plugin.h>
+#include <vlc_vout_window.h>
+#include <vlc_xlib.h>
+#include "vlc_vdpau.h"
+
+typedef struct vdp_instance
+{
+    Display *display;
+    struct vlc_vdp_device device;
+} vdp_instance_t;
+
+static void Close(vlc_decoder_device *device)
+{
+    vdp_instance_t *vi = device->sys;
+
+    vdp_device_destroy(vi->device.vdp, vi->device.device);
+    vdp_destroy_x11(vi->device.vdp);
+    XCloseDisplay(vi->display);
+    free(vi);
+}
+
+static const struct vlc_decoder_device_operations ops = {
+    .close = Close,
+};
+
+static int ScreenNumberOfWindow(Display *dpy, Window w)
+{
+    XWindowAttributes attrs;
+
+    XGetWindowAttributes(dpy, w, &attrs);
+
+    for (int num = 0; num < ScreenCount(dpy); num++)
+        if (RootWindow(dpy, num) == attrs.root)
+            return num;
+
+    return -1;
+}
+
+static int Open(vlc_decoder_device *device, vout_window_t *window)
+{
+    if (window == NULL || window->type != VOUT_WINDOW_TYPE_XID)
+        return VLC_ENOTSUP;
+    if (!vlc_xlib_init(VLC_OBJECT(device)))
+        return VLC_ENOTSUP;
+
+    vdp_instance_t *vi = malloc(sizeof (*vi));
+    if (unlikely(vi == NULL))
+        return VLC_ENOMEM;
+
+    vi->display = XOpenDisplay(window->display.x11);
+    if (vi->display == NULL)
+    {
+        free(vi);
+        return -ENOBUFS;
+    }
+
+    int num = ScreenNumberOfWindow(vi->display, window->handle.xid);
+    if (unlikely(num < 0))
+        goto error;
+
+    VdpStatus err = vdp_create_x11(vi->display, num, &vi->device.vdp,
+                                   &vi->device.device);
+    if (err != VDP_STATUS_OK)
+        goto error;
+
+    const char *infos;
+    err = vdp_get_information_string(vi->device.vdp, &infos);
+    if (err == VDP_STATUS_OK)
+    {
+        if (strstr(infos, "VAAPI") != NULL)
+        {
+            Close(device);
+            return -EACCES;
+        }
+
+        msg_Info(device, "Using %s", infos);
+    }
+
+    device->ops = &ops;
+    device->sys = vi;
+    device->type = VLC_DECODER_DEVICE_VDPAU;
+    device->opaque = &vi->device;
+    return VLC_SUCCESS;
+
+error:
+    XCloseDisplay(vi->display);
+    free(vi);
+    return VLC_EGENERIC;
+}
+
+vlc_module_begin()
+    set_description("VDPAU decoder device")
+    set_callback_dec_device(Open, 3)
+    add_shortcut("vdpau")
+    set_subcategory(SUBCAT_VIDEO_VOUT)
+vlc_module_end()